Skip to main content

datumesh_http/
publish.rs

1//! Publishing stored cells as the mesh (`datumesh-mesh publish-stored`): a JSON lines file of
2//! stored functions and their cells, signed with the mesh's key, put in the store.
3//!
4//! Each line is one of:
5//!
6//! * a descriptor (since 0.3.1, CONTRACT-1.7a H1): `{"descriptor": <function descriptor>}`: the
7//!   descriptor is put with `put_descriptor` and binds nothing. Only a stored function is
8//!   accepted (`code.lang` = `stored`, `code.body` = `""`, `inputs` = `[]`); any other is refused
9//!   at its line. A class later in the file (or in a later run) may name its hash as a method;
10//! * a function: `{"function": "<ns>/<name>", "signature": {...}, "bound": "<instant>"}`: the
11//!   descriptor `{"code": {"body": "", "lang": "stored"}, "inputs": [], "signature": ...}`
12//!   is put and the name bound to it from `bound` (unless it already is);
13//! * a formula function: `{"formula": "<ns>/<name>", "source": "<formula>", "signature":
14//!   {...}, "bound": "<instant>"}`: defined with `datumesh_formula::definition::define_function`
15//!   (its callees as bound at `bound`) and published (`put_descriptor`, `bind_name` from
16//!   `bound`) unless the name is already bound to that function; its cells are evaluated on
17//!   demand by the mesh;
18//! * a class (since 0.2.2, spec-0.5.0, 16.2): `{"class": "<ns>/<name>", "descriptor": {...as
19//!   written...}, "bound": "<instant>"}`: defined with the formula crate's `define_class` (core's
20//!   rules: the class is under one of the mesh's own namespaces, else `E_CLASS_OWNER`; the method
21//!   functions are those bound as of `bound`), `put_class`, and `bind_class` from `bound` unless the
22//!   name is already bound to that class hash. Method functions come first in the file (or in an
23//!   earlier one), `bound` later than theirs. A line with both `class` and `descriptor` is a class
24//!   line;
25//! * a cell: `{"cell": {"fn": "<ns>/<name>", "args": {<name>: <typed scalar>}}, "value": <typed
26//!   scalar> or {"blob": "<file>", "media_type": "<type>"}, "tx"?, "asof"?, "issued"?, "licence",
27//!   "summary"}`: a blob's file (relative to the JSON lines file) is put with `put_blob` and the
28//!   value is `<sha256>:<length>:<media type>`; the receipt is signed with the mesh's key
29//!   (`computed_by` the mesh host, `publisher` the cell's namespace, `valid` its valid-time
30//!   arguments; `issued` defaults to the later of `asof` and the key's start) and stored with
31//!   `put_receipt` and `put_version`. `tx` and `asof` default to the run's clock now (since
32//!   0.3.1, H4), each independently.
33//!
34//! The function of a cell is the one its name is bound to as of the cell's `tx`. Re-running a
35//! file changes nothing (a cell line without `tx` whose latest version states the same thing is
36//! skipped, since 0.3.1): descriptors and blobs are content-addressed, signing is deterministic
37//! (the same receipt, the same hash), and a version already there is skipped. The key is never
38//! logged.
39//!
40//! Rule A (since 0.3.0, CONTRACT-1.8b H1): no binding starts earlier than the mesh clock. The
41//! start of a binding (a name's, a class's, or a class method's `N/m`) is its `bound`, or the
42//! clock's now when `bound` is left out, and a start earlier than now is refused with
43//! [`PublishErrorKind::StartBeforeNow`]. A `--backfill` run ([`PublishOptions::backfill`]) lifts
44//! the start check, and is refused ([`PublishErrorKind::Backfill`]) when a name it would bind has
45//! a binding, or when a namespace of such a name has any binding (store 0.4.2's
46//! `has_bindings_under`). A run that writes appends one line to the log: `{"at": …, "namespaces":
47//! […], "starts": […]}`, a JCS object.
48//!
49//! No partial writes (since 0.3.1, CONTRACT-1.7a H2): the file is read in two passes. The first
50//! plans every write the second one makes, and checks each as the store will: a bind is refused
51//! when its start is not later than the latest binding of its name (the store's, or one planned
52//! earlier in the file, W1b), when the name is of the other kind (a class or a function), or when
53//! its namespace is suspended; a cell's function must be bound as of its `tx` (counting the
54//! earlier binds of the file), its blob must be readable, `tx <= asof <= issued`, and a version
55//! at or after its `tx` must be this receipt. Receipts are signed in the first pass (signing is
56//! deterministic), so their hashes are known. A file that the first pass refuses writes nothing.
57
58use std::collections::{BTreeMap, BTreeSet};
59use std::io::{BufRead, Write};
60use std::path::{Path, PathBuf};
61use std::sync::Arc;
62
63use datumesh_core::json::{self, Value};
64use datumesh_core::{
65    sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey, Type,
66};
67use datumesh_engine::{Clock, SystemClock};
68use datumesh_formula::definition::{Callee, Callees, ClassCallee};
69use datumesh_store::Store;
70
71/// The end of time: a binding as of it is the latest one.
72const END: &str = "9999-12-31T23:59:59.999999Z";
73
74/// What a run put.
75#[non_exhaustive]
76#[derive(Clone, Debug, Default, PartialEq, Eq)]
77pub struct Published {
78    /// Functions newly bound (stored and formula).
79    pub functions: usize,
80    /// Classes newly bound (since 0.2.2).
81    pub classes: usize,
82    /// Cell lines read.
83    pub cells: usize,
84    /// Receipts stored (new or already there).
85    pub receipts: usize,
86    /// Versions newly added.
87    pub versions: usize,
88    /// Blobs put (new or already there).
89    pub blobs: usize,
90    /// Blob bytes put.
91    pub blob_bytes: u64,
92    /// Descriptors newly put by descriptor lines (since 0.3.1); a re-run counts 0.
93    pub descriptors: u64,
94}
95
96/// How a run reads the clock and whether it backfills (CONTRACT-1.8b H1).
97#[derive(Clone)]
98pub struct PublishOptions {
99    /// The mesh clock: the start of a binding left out is its now, and a start earlier than it is
100    /// refused (rule A).
101    pub clock: Arc<dyn Clock + Send + Sync>,
102    /// The operator log of a `--backfill` run (`DATUMESH_BACKFILL_LOG`). `Some` lifts the start
103    /// check, for a file whose names and namespaces have no bindings yet, and appends a line to
104    /// the log for each run that writes.
105    pub backfill: Option<PathBuf>,
106}
107
108impl PublishOptions {
109    /// The system clock, and no backfill.
110    pub fn system() -> PublishOptions {
111        PublishOptions { clock: Arc::new(SystemClock), backfill: None }
112    }
113}
114
115/// Why a line (or the file, at line 0) was refused.
116#[non_exhaustive]
117#[derive(Clone, Debug, PartialEq, Eq)]
118pub enum PublishErrorKind {
119    /// Any other refusal: [`PublishError::detail`] says what.
120    Line,
121    /// A binding's start is earlier than the mesh clock (rule A).
122    StartBeforeNow {
123        /// The name (or class name, or method name) that would bind.
124        name: String,
125        /// Its start.
126        start: String,
127        /// The mesh clock.
128        now: String,
129    },
130    /// A `--backfill` run that would write over a binding, or under a namespace with one.
131    Backfill(String),
132    /// `import-vectors`: a name the vectors would bind is under a namespace the mesh owns.
133    OwnedVectorName(String),
134}
135
136impl std::fmt::Display for PublishErrorKind {
137    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
138        match self {
139            PublishErrorKind::Line => f.write_str("refused"),
140            PublishErrorKind::StartBeforeNow { name, start, now } => {
141                write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
142            }
143            PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
144            PublishErrorKind::OwnedVectorName(name) => {
145                write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
146            }
147        }
148    }
149}
150
151/// Why a run stopped: the line, what was wrong, and its typed kind.
152#[derive(Clone, Debug, PartialEq, Eq)]
153pub struct PublishError {
154    /// The line number (1-based; 0 for the file).
155    pub line: usize,
156    /// Why, in words.
157    pub detail: String,
158    /// Why, typed.
159    pub kind: PublishErrorKind,
160}
161
162impl PublishError {
163    fn at(line: usize, detail: impl Into<String>) -> PublishError {
164        PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
165    }
166
167    pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
168        PublishError { line, detail: kind.to_string(), kind }
169    }
170}
171
172impl std::fmt::Display for PublishError {
173    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
174        write!(f, "line {}: {}", self.line, self.detail)
175    }
176}
177
178impl std::error::Error for PublishError {}
179
180fn s(v: &str) -> Value {
181    Value::String(v.to_string())
182}
183
184/// The namespace of a name: the name minus its last `/` segment.
185fn namespace_of(name: &str) -> &str {
186    name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
187}
188
189/// The host of a name: its first segment.
190fn host_of(name: &str) -> &str {
191    name.split('/').next().unwrap_or("")
192}
193
194/// The definer of a class `name`: its own namespace when the mesh owns the name's host, else the
195/// mesh's host (which no class name under another host has, so core answers `E_CLASS_OWNER`).
196fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
197    let host_of_name = name.split('/').next().unwrap_or("");
198    if owned.iter().any(|o| o == host_of_name) {
199        namespace_of(name)
200    } else {
201        host
202    }
203}
204
205/// The start of a line's binding: its `bound`, or the clock's now when it is left out.
206fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
207    match v.get("bound") {
208        None => Ok(now.to_string()),
209        Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
210    }
211}
212
213/// The bindings and versions a pass one has planned so far: the file's own, which are seen before
214/// the store's.
215#[derive(Default)]
216struct Simulation {
217    /// Function name to the (tx, function hash) of each bind the file would make.
218    functions: BTreeMap<String, Vec<(String, String)>>,
219    /// Class name to the (tx, class hash) of each class bind the file would make.
220    classes: BTreeMap<String, Vec<(String, String)>>,
221    /// Canonical bytes by function hash, of the descriptors the file would store.
222    descriptors: BTreeMap<String, Vec<u8>>,
223    /// Canonical bytes by class hash, of the classes the file would store.
224    class_bytes: BTreeMap<String, Vec<u8>>,
225    /// Cell id to the (tx, receipt hash) of each version the file would add, in tx order.
226    versions: BTreeMap<String, Vec<(String, String)>>,
227}
228
229impl Simulation {
230    /// The latest of `binds`' entries for `name` with tx <= `at`.
231    fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
232        binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
233    }
234}
235
236/// The callees of a definition: the store's bindings, and with `sim` the file's own over them.
237struct Callers<'a, S: Store + ?Sized> {
238    store: &'a S,
239    sim: Option<&'a Simulation>,
240}
241
242impl<S: Store + ?Sized> Callers<'_, S> {
243    fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
244        if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
245            return Ok(Some(b.clone()));
246        }
247        Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
248    }
249
250    fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
251        if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
252            return Ok(Some(b.clone()));
253        }
254        self.store.class(hash).map_err(|e| e.to_string())
255    }
256}
257
258impl<S: Store + ?Sized> Callees for Callers<'_, S> {
259    fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
260        let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
261        let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
262        let (fn_hash, start) = match (stored, simmed) {
263            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
264            (Some(b), _) => (b.fn_hash, b.tx),
265            (None, Some((t, h))) => (h, t),
266            (None, None) => return Ok(None),
267        };
268        let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
269        let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
270        Ok(Some(Callee { fn_hash, start, descriptor }))
271    }
272
273    fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
274        let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
275        let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
276        let (class_hash, start) = match (stored, simmed) {
277            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
278            (Some(b), _) => (b.class_hash, b.tx),
279            (None, Some((t, h))) => (h, t),
280            (None, None) => return Ok(None),
281        };
282        let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
283        let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
284        Ok(Some(ClassCallee { class_hash, start, descriptor }))
285    }
286
287    fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
288        let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
289        ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
290    }
291
292    fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
293        let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
294        json::parse(&bytes).map(Some).map_err(|e| e.to_string())
295    }
296}
297
298/// The function hash `name` means at the end of time: the file's own over the store's.
299fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
300    if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
301        return Ok(Some(h));
302    }
303    store.resolve_name(name, END).map_err(|e| e.to_string())
304}
305
306/// The class hash `name` means at the end of time: the file's own over the store's.
307fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
308    if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
309        return Ok(Some(h));
310    }
311    store.resolve_class(name, END).map_err(|e| e.to_string())
312}
313
314/// The function hash `name` means as of `at`, counting the file's earlier binds.
315fn function_at<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, at: &str) -> Result<Option<String>, String> {
316    let stored = store.resolve_binding(name, at).map_err(|e| e.to_string())?;
317    let simmed = Simulation::latest(&sim.functions, name, at);
318    Ok(match (stored, simmed) {
319        (Some(b), Some((t, h))) if t >= b.tx => Some(h),
320        (Some(b), _) => Some(b.fn_hash),
321        (None, Some((_, h))) => Some(h),
322        (None, None) => None,
323    })
324}
325
326/// The latest binding tx of `name` (a class's, or a function's), the store's or the file's.
327fn latest_bind<S: Store + ?Sized>(
328    store: &S,
329    sim: &Simulation,
330    name: &str,
331    class: bool,
332) -> Result<Option<String>, String> {
333    let stored: Option<String> = if class {
334        store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
335    } else {
336        store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
337    };
338    let binds = if class { &sim.classes } else { &sim.functions };
339    let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
340    Ok(match (stored, planned) {
341        (Some(a), Some(b)) => Some(a.max(b)),
342        (a, b) => a.or(b),
343    })
344}
345
346/// Whether `name` is bound as the other kind: a function for a class, a class for a function.
347fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
348    if class {
349        Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
350    } else {
351        Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
352            || sim.classes.contains_key(name))
353    }
354}
355
356/// The latest version of a cell, the store's or the file's: its tx and receipt hash.
357fn cell_latest<S: Store + ?Sized>(store: &S, sim: &Simulation, id: &str) -> Result<Option<(String, String)>, String> {
358    let stored = store.latest(id).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
359    let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
360    Ok(match (stored, planned) {
361        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
362        (a, b) => a.or(b),
363    })
364}
365
366/// The receipt hash of a cell's version as of `at`, the store's or the file's.
367fn cell_version_asof<S: Store + ?Sized>(
368    store: &S,
369    sim: &Simulation,
370    id: &str,
371    at: &str,
372) -> Result<Option<String>, String> {
373    let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
374    let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
375    let latest = match (stored, planned) {
376        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
377        (a, b) => a.or(b),
378    };
379    Ok(latest.map(|x| x.1))
380}
381
382/// Whether a namespace is suspended (the store refuses binds and class puts under it).
383fn suspended<S: Store + ?Sized>(store: &S, host: &str, line: usize) -> Result<(), PublishError> {
384    match store.namespace(host) {
385        Ok(Some(n)) if n.suspended_at.is_some() => {
386            Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
387        }
388        Ok(_) => Ok(()),
389        Err(e) => Err(PublishError::at(line, e.to_string())),
390    }
391}
392
393/// A binding a run would make: the line that makes it and its name.
394struct Bind {
395    line: usize,
396    name: String,
397}
398
399/// What a run writes, for the backfill log: the namespaces and starts of its binds.
400#[derive(Default)]
401struct Written {
402    namespaces: BTreeSet<String>,
403    starts: BTreeSet<String>,
404}
405
406impl Written {
407    fn bound(&mut self, name: &str, start: &str) {
408        self.namespaces.insert(namespace_of(name).to_string());
409        self.starts.insert(start.to_string());
410    }
411}
412
413/// The per-run values both passes use.
414struct Ctx<'a> {
415    host: &'a str,
416    owned: &'a [String],
417    /// The start of the mesh's key.
418    key_from: &'a str,
419    /// The clock's now, read once for the run.
420    now: &'a str,
421    /// The directory of the file (blobs are relative to it).
422    base: &'a Path,
423}
424
425/// One write the second pass makes, planned (and checked) by the first.
426enum Op {
427    /// A descriptor line: put; counted when newly put.
428    Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
429    /// A function (stored or formula): put its descriptor, and bind the name when `bind`.
430    Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
431    /// A class: put it, and bind it (its methods with it, in the store) when `bind`.
432    Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
433    /// A cell: its blob, its receipt, and the version when `version`.
434    Cell { tx: String, id: String, hash: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
435}
436
437/// The lines of a file, numbered, without the blank ones.
438fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
439    let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
440    let mut out = Vec::new();
441    for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
442        let n = i + 1;
443        let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
444        if !line.trim().is_empty() {
445            out.push((n, line));
446        }
447    }
448    Ok(out)
449}
450
451/// Pass one: plans the writes of the lines and checks each as the store will (CONTRACT-1.7a H2).
452struct Planner<'a, S: Store + ?Sized> {
453    store: &'a S,
454    key: &'a SigningKey,
455    ctx: Ctx<'a>,
456    backfill: bool,
457    sim: Simulation,
458    ops: Vec<Op>,
459    binds: Vec<Bind>,
460}
461
462impl<S: Store + ?Sized> Planner<'_, S> {
463    fn callers(&self) -> Callers<'_, S> {
464        Callers { store: self.store, sim: Some(&self.sim) }
465    }
466
467    fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
468        for (n, text) in lines {
469            let n = *n;
470            let err = |d: String| PublishError::at(n, d);
471            let v = json::parse(text.as_bytes()).map_err(|e| err(e.to_string()))?;
472            if let Some(name) = v.get("class").and_then(Value::as_str) {
473                self.class_line(n, name, &v)?;
474            } else if let Some(d) = v.get("descriptor") {
475                self.descriptor_line(n, d)?;
476            } else if let Some(name) = v.get("function").and_then(Value::as_str) {
477                let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
478                let start = bound_of(&v, n, self.ctx.now)?;
479                let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
480                self.function(n, name, start, fd.hash(), fd.canonical())?;
481            } else if let Some(name) = v.get("formula").and_then(Value::as_str) {
482                let source =
483                    v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
484                let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
485                let start = bound_of(&v, n, self.ctx.now)?;
486                let def = datumesh_formula::definition::define_function(
487                    name,
488                    source,
489                    sig,
490                    &Default::default(),
491                    &self.callers(),
492                    &start,
493                )
494                .map_err(|e| err(format!("define {name}: {e}")))?;
495                self.function(n, name, start, def.fn_hash, def.bytes)?;
496            } else if let Some(cell_v) = v.get("cell") {
497                self.cell_line(n, &v, cell_v)?;
498            } else {
499                return Err(err("neither a function nor a cell".into()));
500            }
501        }
502        Ok(())
503    }
504
505    /// `{"descriptor": ...}`: a stored function's descriptor, put and bound to nothing.
506    fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
507        let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
508        if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
509            return Err(PublishError::at(
510                n,
511                "a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
512            ));
513        }
514        let (hash, bytes) = (fd.hash(), fd.canonical());
515        self.sim.descriptors.insert(hash.clone(), bytes.clone());
516        self.ops.push(Op::Descriptor { hash, bytes, counted: true });
517        Ok(())
518    }
519
520    /// A function's descriptor (stored or formula): put, and bound when its name does not mean it.
521    fn function(
522        &mut self,
523        n: usize,
524        name: &str,
525        start: String,
526        hash: String,
527        bytes: Vec<u8>,
528    ) -> Result<(), PublishError> {
529        self.sim.descriptors.insert(hash.clone(), bytes.clone());
530        let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
531        let bind = current.as_deref() != Some(hash.as_str());
532        if bind {
533            self.check_bind(n, name, &start, false)?;
534            self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
535        }
536        self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
537        Ok(())
538    }
539
540    fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
541        let err = |d: String| PublishError::at(n, d);
542        let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
543        let start = bound_of(v, n, self.ctx.now)?;
544        let def = datumesh_formula::definition::define_class(
545            name,
546            &descriptor.to_jcs(),
547            definer(name, self.ctx.host, self.ctx.owned),
548            &self.callers(),
549            &start,
550        )
551        .map_err(|e| err(format!("define {name}: {e}")))?;
552        // put_class refuses under a suspended steward namespace, whether or not the name binds. A
553        // namespace row is a host's (9.9), so the steward namespace's host is asked: a steward with
554        // a path (datumesh.com/scripture) is not a host and Store::namespace refuses it.
555        suspended(self.store, host_of(&def.descriptor.steward.namespace), n)?;
556        let hash = def.class_hash.clone();
557        let methods: Vec<(String, String)> =
558            def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
559        let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
560        if bind {
561            self.check_bind(n, name, &start, true)?;
562            // bind_class binds every method in the same transaction (the store's rule).
563            for (mname, _) in &methods {
564                self.check_bind(n, mname, &start, false)?;
565            }
566            self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
567            for (mname, fh) in &methods {
568                self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
569            }
570        }
571        self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
572        self.ops.push(Op::Class {
573            name: name.to_string(),
574            start,
575            hash,
576            bytes: def.bytes,
577            bind,
578            methods: methods.into_iter().map(|(m, _)| m).collect(),
579        });
580        Ok(())
581    }
582
583    /// One bind a line would make: rule A, the other kind, the namespace, and the latest binding.
584    fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
585        if !self.backfill && start < self.ctx.now {
586            return Err(PublishError::typed(
587                line,
588                PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
589            ));
590        }
591        let err = |d: String| PublishError::at(line, d);
592        if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
593            let kind = if class { "function" } else { "class" };
594            return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
595        }
596        suspended(self.store, host_of(name), line)?;
597        if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
598            if start <= latest.as_str() {
599                return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
600            }
601        }
602        self.binds.push(Bind { line, name: name.into() });
603        Ok(())
604    }
605
606    fn cell_line(&mut self, n: usize, v: &Value, cell_v: &Value) -> Result<(), PublishError> {
607        let err = |d: String| PublishError::at(n, d);
608        let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
609        let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
610        let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
611        let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
612        let (licence, summary) = (required("licence")?, required("summary")?);
613        // tx and asof default to the run's clock now, each on its own (H4).
614        let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
615        let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
616        if tx > asof {
617            return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
618        }
619        let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
620        if asof > issued {
621            return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
622        }
623        let fn_hash = function_at(self.store, &self.sim, name, &tx)
624            .map_err(err)?
625            .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
626        let d = self
627            .callers()
628            .function_bytes(&fn_hash)
629            .map_err(err)?
630            .ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
631        let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
632        let mut args = Args::new();
633        for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
634            args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
635        }
636        let valid: Args = args
637            .iter()
638            .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
639            .map(|(k, a)| (k.clone(), a.clone()))
640            .collect();
641        let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
642        let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
643            Some(file) => {
644                let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
645                let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
646                let sha = sha256_hex(&bytes);
647                let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
648                    .map_err(|e| err(e.to_string()))?;
649                (scalar, Some(bytes))
650            }
651            None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
652        };
653        let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
654        let r = ReceiptBuilder::new()
655            .cell(cell.clone())
656            .value(value)
657            .valid(valid)
658            .asof(asof.as_str())
659            .tx(tx.as_str())
660            .issued(issued.as_str())
661            .publisher(ns)
662            .signer_host(self.ctx.host)
663            .licence(licence.as_str())
664            .summary(summary.as_str())
665            .sign(self.key)
666            .map_err(|e| err(e.to_string()))?;
667        let receipt = receipt_json(&r);
668        let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
669        let id = cell.id();
670        // A line without tx takes the run's clock, so a re-run at a later clock would sign the same
671        // fact again as a new version. Re-running a file changes nothing: when the cell's latest
672        // version is the store's and states the same thing (type, value, valid, licence, summary),
673        // the line is skipped. A changed value is a new version, a correction.
674        if field("tx").is_none() {
675            if let Some((_, latest)) = cell_latest(self.store, &self.sim, &id).map_err(err)? {
676                if let Some(old) = self.store.receipt(&latest) {
677                    let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
678                    let new = json::parse(&receipt).map_err(|e| err(e.to_string()))?;
679                    if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
680                        return Ok(());
681                    }
682                }
683            }
684        }
685        let version = match cell_latest(self.store, &self.sim, &id).map_err(err)? {
686            Some((latest_tx, _)) if latest_tx.as_str() >= tx.as_str() => {
687                if cell_version_asof(self.store, &self.sim, &id, &tx).map_err(err)?.as_deref() != Some(hash.as_str()) {
688                    return Err(err(format!("{id} has a version at or after {tx} that is not this receipt")));
689                }
690                false
691            }
692            _ => {
693                self.sim.versions.entry(id.clone()).or_default().push((tx.clone(), hash.clone()));
694                true
695            }
696        };
697        self.ops.push(Op::Cell { tx, id, hash, receipt, blob, version });
698        Ok(())
699    }
700}
701
702/// The stored descriptor of a function line's signature.
703fn stored_value(signature: &Value) -> Value {
704    Value::Object(vec![
705        ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
706        ("inputs".into(), Value::Array(vec![])),
707        ("signature".into(), signature.clone()),
708    ])
709}
710
711/// The backfill's checks (since 0.3.0 with store 0.4.2): no name the file binds has a binding, and
712/// no namespace of one has any binding (`has_bindings_under`).
713fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
714    for b in binds {
715        let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
716        if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
717            return Err(refuse(format!("{} has a binding", b.name)));
718        }
719        let ns = namespace_of(&b.name);
720        if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
721            return Err(refuse(format!("{ns} has bindings")));
722        }
723    }
724    Ok(())
725}
726
727/// Appends a run's line to the backfill log (created with mode 0600).
728fn append_log(log: &Path, written: &Written) -> Result<(), String> {
729    let line = Value::Object(vec![
730        ("at".into(), s(&SystemClock.now())),
731        ("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
732        ("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
733    ])
734    .to_jcs();
735    let mut opts = std::fs::OpenOptions::new();
736    opts.create(true).append(true);
737    #[cfg(unix)]
738    {
739        use std::os::unix::fs::OpenOptionsExt;
740        opts.mode(0o600);
741    }
742    let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
743    let mut bytes = line;
744    bytes.push(b'\n');
745    f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
746}
747
748/// Pass two: the writes the first pass planned, in order. Nothing here can refuse a planned
749/// write, so a refused file has written nothing.
750fn write<S: Store + ?Sized>(store: &S, ops: &[Op]) -> Result<(Published, Written), PublishError> {
751    let mut out = Published::default();
752    let mut written = Written::default();
753    let err = |d: String| PublishError::at(0, d);
754    for op in ops {
755        match op {
756            Op::Descriptor { hash, bytes, counted } => {
757                let existed = store.descriptor(hash).is_some();
758                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
759                if *counted && !existed {
760                    out.descriptors += 1;
761                }
762            }
763            Op::Function { name, start, hash, bytes, bind } => {
764                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
765                if *bind {
766                    store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
767                    out.functions += 1;
768                    written.bound(name, start);
769                }
770            }
771            Op::Class { name, start, hash, bytes, bind, methods } => {
772                store.put_class(bytes).map_err(|e| err(e.to_string()))?;
773                if *bind {
774                    store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
775                    out.classes += 1;
776                    written.bound(name, start);
777                    for m in methods {
778                        written.bound(m, start);
779                    }
780                }
781            }
782            Op::Cell { tx, id, hash, receipt, blob, version } => {
783                if let Some(bytes) = blob {
784                    store.put_blob(bytes).map_err(|e| err(e.to_string()))?;
785                    out.blobs += 1;
786                    out.blob_bytes += bytes.len() as u64;
787                }
788                store.put_receipt(receipt).map_err(|e| err(e.to_string()))?;
789                out.cells += 1;
790                out.receipts += 1;
791                if *version {
792                    store.put_version(id, tx, hash).map_err(|e| err(e.to_string()))?;
793                    out.versions += 1;
794                }
795            }
796        }
797    }
798    Ok((out, written))
799}
800
801/// The receipt JSON of a signed receipt: its payload's members and `sig`.
802pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
803    let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
804    if let Ok(Value::Object(m)) = json::parse(r.payload()) {
805        members.extend(m);
806    }
807    Value::Object(members).to_jcs()
808}
809
810/// Publishes the file at `path` into `store` as the mesh `host` signing with `key` (see the
811/// module documentation). The mesh's own namespaces are `host` alone: see [`publish_stored_in`].
812/// The clock is the system clock, and there is no backfill.
813pub fn publish_stored<S: Store + ?Sized>(
814    store: &S,
815    key: &SigningKey,
816    host: &str,
817    path: &Path,
818) -> Result<Published, PublishError> {
819    publish_stored_in(store, key, host, &[host.to_string()], path)
820}
821
822/// As [`publish_stored`], for a mesh that owns the namespaces (hosts) `owned` (`Config::namespaces`):
823/// a class line is accepted for a class under one of them, and refused with `E_CLASS_OWNER` (in the
824/// [`PublishError::detail`]) under any other. Since 0.2.2. The system clock, and no backfill.
825pub fn publish_stored_in<S: Store + ?Sized>(
826    store: &S,
827    key: &SigningKey,
828    host: &str,
829    owned: &[String],
830    path: &Path,
831) -> Result<Published, PublishError> {
832    publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
833}
834
835/// As [`publish_stored_in`], with the run's clock and backfill (CONTRACT-1.8b H1, rule A).
836///
837/// Errors: a line refused by its checks, or a binding whose start is earlier than the clock
838/// ([`PublishErrorKind::StartBeforeNow`]), or a backfill refused ([`PublishErrorKind::Backfill`]).
839/// Nothing is written when the first pass refuses (CONTRACT-1.7a H2).
840pub fn publish_stored_with<S: Store + ?Sized>(
841    store: &S,
842    key: &SigningKey,
843    host: &str,
844    owned: &[String],
845    path: &Path,
846    opts: &PublishOptions,
847) -> Result<Published, PublishError> {
848    let lines = read_lines(path)?;
849    // The key's start: the earliest `issued`.
850    let key_from = store
851        .namespace(host)
852        .ok()
853        .flatten()
854        .and_then(|n| json::parse(&n.keydoc).ok())
855        .and_then(|d| match d.get("keys") {
856            Some(Value::Array(ks)) => ks
857                .iter()
858                .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
859                .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
860            _ => None,
861        })
862        .ok_or_else(|| {
863            PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
864        })?;
865    let now = opts.clock.now();
866    let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
867    let mut planner = Planner {
868        store,
869        key,
870        ctx,
871        backfill: opts.backfill.is_some(),
872        sim: Simulation::default(),
873        ops: Vec::new(),
874        binds: Vec::new(),
875    };
876    planner.run(&lines)?;
877    let Planner { ops, binds, .. } = planner;
878    if opts.backfill.is_some() {
879        backfill_check(store, &binds)?;
880    }
881    let (out, written) = write(store, &ops)?;
882    if let Some(log) = &opts.backfill {
883        if out.functions + out.classes + out.versions > 0 {
884            append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
885        }
886    }
887    Ok(out)
888}