Skip to main content

datumesh_http/
publish.rs

1//! Publishing stored cells as the mesh (`datumesh-mesh publish-stored`): a JSON lines file of
2//! stored functions and their cells, signed with the mesh's key, put in the store.
3//!
4//! Each line is one of:
5//!
6//! * a descriptor (since 0.3.1, CONTRACT-1.7a H1): `{"descriptor": <function descriptor>}`: the
7//!   descriptor is put with `put_descriptor` and binds nothing. Only a stored function is
8//!   accepted (`code.lang` = `stored`, `code.body` = `""`, `inputs` = `[]`); any other is refused
9//!   at its line. A class later in the file (or in a later run) may name its hash as a method;
10//! * a function: `{"function": "<ns>/<name>", "signature": {...}, "bound": "<instant>"}`: the
11//!   descriptor `{"code": {"body": "", "lang": "stored"}, "inputs": [], "signature": ...}`
12//!   is put and the name bound to it from `bound` (unless it already is);
13//! * a formula function: `{"formula": "<ns>/<name>", "source": "<formula>", "signature":
14//!   {...}, "bound": "<instant>"}`: defined with `datumesh_formula::definition::define_function`
15//!   (its callees as bound at `bound`) and published (`put_descriptor`, `bind_name` from
16//!   `bound`) unless the name is already bound to that function; its cells are evaluated on
17//!   demand by the mesh;
18//! * a class (since 0.2.2, spec-0.5.0, 16.2): `{"class": "<ns>/<name>", "descriptor": {...as
19//!   written...}, "bound": "<instant>"}`: defined with the formula crate's `define_class` (core's
20//!   rules: the class is under one of the mesh's own namespaces, else `E_CLASS_OWNER`; the method
21//!   functions are those bound as of `bound`), `put_class`, and `bind_class` from `bound` unless the
22//!   name is already bound to that class hash. Method functions come first in the file (or in an
23//!   earlier one), `bound` later than theirs. A line with both `class` and `descriptor` is a class
24//!   line;
25//! * a cell: `{"cell": {"fn": "<ns>/<name>", "args": {<name>: <typed scalar>}}, "value": <typed
26//!   scalar> or {"blob": "<file>", "media_type": "<type>"}, "tx"?, "asof"?, "issued"?, "licence",
27//!   "summary"}`: a blob's file (relative to the JSON lines file) is put with `put_blob` and the
28//!   value is `<sha256>:<length>:<media type>`; the receipt is signed with the mesh's key
29//!   (`computed_by` the mesh host, `publisher` the cell's namespace, `valid` its valid-time
30//!   arguments; `issued` defaults to the later of `asof` and the key's start) and stored with
31//!   `put_receipt` and `put_version`. `tx` and `asof` default to the run's clock now (since
32//!   0.3.1, H4), each independently.
33//!
34//! The function of a cell is the one its name is bound to as of the cell's `tx`. Re-running a
35//! file changes nothing (a cell line without `tx` whose latest version states the same thing is
36//! skipped, since 0.3.1): descriptors and blobs are content-addressed, signing is deterministic
37//! (the same receipt, the same hash), and a version already there is skipped. The key is never
38//! logged.
39//!
40//! Rule A (since 0.3.0, CONTRACT-1.8b H1): no binding starts earlier than the mesh clock. The
41//! start of a binding (a name's, a class's, or a class method's `N/m`) is its `bound`, or the
42//! clock's now when `bound` is left out, and a start earlier than now is refused with
43//! [`PublishErrorKind::StartBeforeNow`]. A `--backfill` run ([`PublishOptions::backfill`]) lifts
44//! the start check, and is refused ([`PublishErrorKind::Backfill`]) when a name it would bind has
45//! a binding, or when a namespace of such a name has any binding (store 0.4.2's
46//! `has_bindings_under`). A run that writes appends one line to the log: `{"at": …, "namespaces":
47//! […], "starts": […]}`, a JCS object.
48//!
49//! No partial writes (since 0.3.1, CONTRACT-1.7a H2): the file is read in two passes. The first
50//! plans every write the second one makes, and checks each as the store will: a bind is refused
51//! when its start is not later than the latest binding of its name (the store's, or one planned
52//! earlier in the file, W1b), when the name is of the other kind (a class or a function), or when
53//! its namespace is suspended; a cell's function must be bound as of its `tx` (counting the
54//! earlier binds of the file), its blob must be readable, `tx <= asof <= issued`, and a version
55//! at or after its `tx` must be this receipt. Receipts are signed in the first pass (signing is
56//! deterministic), so their hashes are known. A file that the first pass refuses writes nothing.
57
58use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
59use std::io::{BufRead, Write};
60use std::path::{Path, PathBuf};
61use std::sync::atomic::{AtomicBool, Ordering};
62use std::sync::Arc;
63
64use datumesh_core::json::{self, Value};
65use datumesh_core::{
66    sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey, Type,
67};
68use datumesh_engine::{Clock, SystemClock};
69use datumesh_formula::definition::{Callee, Callees, ClassCallee};
70use datumesh_store::{Binding, CellWrite, Namespace, Store};
71
72use crate::testrun::{self, run_test, Overlay, Ran, RowWrite, TestSpec};
73
74use crate::config::ConfigError;
75
76/// The end of time: a binding as of it is the latest one.
77const END: &str = "9999-12-31T23:59:59.999999Z";
78
79/// Cell lines planned together, and cell writes per `put_cells` call (CONTRACT-perf H1, H2).
80const CHUNK: usize = 1_000;
81
82/// The environment variable of the publish lanes (CONTRACT-perf H5), read once per run.
83const LANES_VAR: &str = "DATUMESH_PUBLISH_LANES";
84
85/// The lanes of a run (CONTRACT-perf H5): `DATUMESH_PUBLISH_LANES` (a positive integer), else
86/// `min(4, available_parallelism)`, at least 1. An invalid value refuses the run before any write.
87fn publish_lanes() -> Result<usize, PublishError> {
88    let bad = |detail: String| PublishError::at(0, ConfigError::new(LANES_VAR, detail).to_string());
89    match std::env::var(LANES_VAR) {
90        Err(std::env::VarError::NotPresent) => Ok(std::thread::available_parallelism().map_or(1, |n| n.get()).min(4)),
91        Ok(v) => match v.parse::<usize>() {
92            Ok(n) if n > 0 => Ok(n),
93            _ => Err(bad(format!("must be a positive integer, not {v:?}"))),
94        },
95        Err(e) => Err(bad(e.to_string())),
96    }
97}
98
99/// What a run put.
100#[non_exhaustive]
101#[derive(Clone, Debug, Default, PartialEq, Eq)]
102pub struct Published {
103    /// Functions newly bound (stored and formula).
104    pub functions: usize,
105    /// Classes newly bound (since 0.2.2).
106    pub classes: usize,
107    /// Cell lines read.
108    pub cells: usize,
109    /// Receipts stored (new or already there).
110    pub receipts: usize,
111    /// Versions newly added.
112    pub versions: usize,
113    /// Blobs put (new or already there).
114    pub blobs: usize,
115    /// Blob bytes put.
116    pub blob_bytes: u64,
117    /// Descriptors newly put by descriptor lines (since 0.3.1); a re-run counts 0.
118    pub descriptors: u64,
119}
120
121/// How a run reads the clock and whether it backfills (CONTRACT-1.8b H1).
122#[derive(Clone)]
123pub struct PublishOptions {
124    /// The mesh clock: the start of a binding left out is its now, and a start earlier than it is
125    /// refused (rule A).
126    pub clock: Arc<dyn Clock + Send + Sync>,
127    /// The operator log of a `--backfill` run (`DATUMESH_BACKFILL_LOG`). `Some` lifts the start
128    /// check, for a file whose names and namespaces have no bindings yet, and appends a line to
129    /// the log for each run that writes.
130    pub backfill: Option<PathBuf>,
131}
132
133impl PublishOptions {
134    /// The system clock, and no backfill.
135    pub fn system() -> PublishOptions {
136        PublishOptions { clock: Arc::new(SystemClock), backfill: None }
137    }
138}
139
140/// Why a line (or the file, at line 0) was refused.
141#[non_exhaustive]
142#[derive(Clone, Debug, PartialEq, Eq)]
143pub enum PublishErrorKind {
144    /// Any other refusal: [`PublishError::detail`] says what.
145    Line,
146    /// A binding's start is earlier than the mesh clock (rule A).
147    StartBeforeNow {
148        /// The name (or class name, or method name) that would bind.
149        name: String,
150        /// Its start.
151        start: String,
152        /// The mesh clock.
153        now: String,
154    },
155    /// A `--backfill` run that would write over a binding, or under a namespace with one.
156    Backfill(String),
157    /// `import-vectors`: a name the vectors would bind is under a namespace the mesh owns.
158    OwnedVectorName(String),
159    /// The public rule (D8, Build 2 Part 3, `publish_stored_checked`): a computed function's tests
160    /// do not cover the derived cases or every branch. The text lists what is missing.
161    TestsMissing(String),
162    /// The public rule: a test of a computed function fails. The text names the case and the
163    /// numbers.
164    TestsFailing(String),
165}
166
167impl std::fmt::Display for PublishErrorKind {
168    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
169        match self {
170            PublishErrorKind::Line => f.write_str("refused"),
171            PublishErrorKind::StartBeforeNow { name, start, now } => {
172                write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
173            }
174            PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
175            PublishErrorKind::OwnedVectorName(name) => {
176                write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
177            }
178            PublishErrorKind::TestsMissing(text) => write!(f, "E_TESTS_MISSING: {text}"),
179            PublishErrorKind::TestsFailing(text) => write!(f, "E_TESTS_FAILING: {text}"),
180        }
181    }
182}
183
184/// Why a run stopped: the line, what was wrong, and its typed kind.
185#[derive(Clone, Debug, PartialEq, Eq)]
186pub struct PublishError {
187    /// The line number (1-based; 0 for the file).
188    pub line: usize,
189    /// Why, in words.
190    pub detail: String,
191    /// Why, typed.
192    pub kind: PublishErrorKind,
193}
194
195impl PublishError {
196    fn at(line: usize, detail: impl Into<String>) -> PublishError {
197        PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
198    }
199
200    pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
201        PublishError { line, detail: kind.to_string(), kind }
202    }
203}
204
205impl std::fmt::Display for PublishError {
206    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
207        write!(f, "line {}: {}", self.line, self.detail)
208    }
209}
210
211impl std::error::Error for PublishError {}
212
213fn s(v: &str) -> Value {
214    Value::String(v.to_string())
215}
216
217/// The namespace of a name: the name minus its last `/` segment.
218fn namespace_of(name: &str) -> &str {
219    name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
220}
221
222/// The host of a name: its first segment.
223fn host_of(name: &str) -> &str {
224    name.split('/').next().unwrap_or("")
225}
226
227/// The definer of a class `name`: its own namespace when the mesh owns the name's host, else the
228/// mesh's host (which no class name under another host has, so core answers `E_CLASS_OWNER`).
229fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
230    let host_of_name = name.split('/').next().unwrap_or("");
231    if owned.iter().any(|o| o == host_of_name) {
232        namespace_of(name)
233    } else {
234        host
235    }
236}
237
238/// The start of a line's binding: its `bound`, or the clock's now when it is left out.
239fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
240    match v.get("bound") {
241        None => Ok(now.to_string()),
242        Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
243    }
244}
245
246/// The bindings and versions a pass one has planned so far: the file's own, which are seen before
247/// the store's.
248#[derive(Default)]
249struct Simulation {
250    /// Function name to the (tx, function hash) of each bind the file would make.
251    functions: BTreeMap<String, Vec<(String, String)>>,
252    /// Class name to the (tx, class hash) of each class bind the file would make.
253    classes: BTreeMap<String, Vec<(String, String)>>,
254    /// Canonical bytes by function hash, of the descriptors the file would store.
255    descriptors: BTreeMap<String, Vec<u8>>,
256    /// Canonical bytes by class hash, of the classes the file would store.
257    class_bytes: BTreeMap<String, Vec<u8>>,
258    /// Cell id to the (tx, receipt hash) of each version the file would add, in tx order.
259    versions: BTreeMap<String, Vec<(String, String)>>,
260}
261
262impl Simulation {
263    /// The latest of `binds`' entries for `name` with tx <= `at`.
264    fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
265        binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
266    }
267}
268
269/// The callees of a definition: the store's bindings, and with `sim` the file's own over them.
270struct Callers<'a, S: Store + ?Sized> {
271    store: &'a S,
272    sim: Option<&'a Simulation>,
273}
274
275impl<S: Store + ?Sized> Callers<'_, S> {
276    fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
277        if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
278            return Ok(Some(b.clone()));
279        }
280        Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
281    }
282
283    fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
284        if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
285            return Ok(Some(b.clone()));
286        }
287        self.store.class(hash).map_err(|e| e.to_string())
288    }
289}
290
291impl<S: Store + ?Sized> Callees for Callers<'_, S> {
292    fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
293        let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
294        let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
295        let (fn_hash, start) = match (stored, simmed) {
296            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
297            (Some(b), _) => (b.fn_hash, b.tx),
298            (None, Some((t, h))) => (h, t),
299            (None, None) => return Ok(None),
300        };
301        let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
302        let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
303        Ok(Some(Callee { fn_hash, start, descriptor }))
304    }
305
306    fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
307        let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
308        let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
309        let (class_hash, start) = match (stored, simmed) {
310            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
311            (Some(b), _) => (b.class_hash, b.tx),
312            (None, Some((t, h))) => (h, t),
313            (None, None) => return Ok(None),
314        };
315        let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
316        let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
317        Ok(Some(ClassCallee { class_hash, start, descriptor }))
318    }
319
320    fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
321        let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
322        ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
323    }
324
325    fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
326        let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
327        json::parse(&bytes).map(Some).map_err(|e| e.to_string())
328    }
329}
330
331/// What a run reads from the store again and again, kept for the run (CONTRACT-perf H1). The
332/// store does not change during the first pass, so each answer is the same whenever it is asked,
333/// and a failed read is kept as its message, which is what the check that asks it reports. A
334/// prefetch that fails keeps nothing: each line then asks for itself and fails at its own line.
335#[derive(Default)]
336struct Reads {
337    bindings: HashMap<(String, String), Result<Option<Binding>, String>>,
338    descriptors: HashMap<String, Option<Vec<u8>>>,
339    namespaces: HashMap<String, Result<Option<Namespace>, String>>,
340    /// The store's latest version of a cell (tx, receipt hash), prefetched per chunk.
341    latest_of: HashMap<String, Option<(String, String)>>,
342    /// The store's receipt, prefetched for the latest receipts a skip check reads.
343    receipt_of: HashMap<String, Option<Vec<u8>>>,
344}
345
346impl Reads {
347    fn binding<S: Store + ?Sized>(&mut self, store: &S, name: &str, at: &str) -> Result<Option<Binding>, String> {
348        let key = (name.to_string(), at.to_string());
349        if let Some(r) = self.bindings.get(&key) {
350            return r.clone();
351        }
352        let r = store.resolve_binding(name, at).map_err(|e| e.to_string());
353        self.bindings.insert(key, r.clone());
354        r
355    }
356
357    fn descriptor<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
358        if let Some(b) = self.descriptors.get(hash) {
359            return b.clone();
360        }
361        let b = store.descriptor(hash).map(|b| b.into_owned());
362        self.descriptors.insert(hash.to_string(), b.clone());
363        b
364    }
365
366    fn namespace<S: Store + ?Sized>(&mut self, store: &S, host: &str) -> Result<Option<Namespace>, String> {
367        if let Some(r) = self.namespaces.get(host) {
368            return r.clone();
369        }
370        let r = store.namespace(host).map_err(|e| e.to_string());
371        self.namespaces.insert(host.to_string(), r.clone());
372        r
373    }
374
375    /// The store's latest version of `id`: prefetched, else asked now.
376    fn latest<S: Store + ?Sized>(&mut self, store: &S, id: &str) -> Result<Option<(String, String)>, String> {
377        if let Some(l) = self.latest_of.get(id) {
378            return Ok(l.clone());
379        }
380        store.latest(id).map(|v| v.map(|v| (v.tx, v.receipt_hash))).map_err(|e| e.to_string())
381    }
382
383    /// The stored receipt `hash`, as `receipt` answers: prefetched, else asked now.
384    fn receipt<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
385        if let Some(b) = self.receipt_of.get(hash) {
386            return b.clone();
387        }
388        store.receipt(hash).map(|b| b.into_owned())
389    }
390
391    /// One `latest_many` for the ids not read yet.
392    fn prefetch_latest<S: Store + ?Sized>(&mut self, store: &S, ids: &[&str]) {
393        let mut seen = HashSet::new();
394        let missing: Vec<&str> =
395            ids.iter().copied().filter(|id| !self.latest_of.contains_key(*id) && seen.insert(*id)).collect();
396        if missing.is_empty() {
397            return;
398        }
399        if let Ok(found) = store.latest_many(&missing) {
400            for (id, v) in missing.iter().zip(found) {
401                self.latest_of.insert(id.to_string(), v.map(|v| (v.tx, v.receipt_hash)));
402            }
403        }
404    }
405
406    /// One `receipts_many` for the hashes not read yet.
407    fn prefetch_receipts<S: Store + ?Sized>(&mut self, store: &S, hashes: &[&str]) {
408        let mut seen = HashSet::new();
409        let missing: Vec<&str> =
410            hashes.iter().copied().filter(|h| !self.receipt_of.contains_key(*h) && seen.insert(*h)).collect();
411        if missing.is_empty() {
412            return;
413        }
414        if let Ok(found) = store.receipts_many(&missing) {
415            for (h, b) in missing.iter().zip(found) {
416                self.receipt_of.insert(h.to_string(), b);
417            }
418        }
419    }
420}
421
422/// `f` over `items` on up to `available_parallelism()` threads; the results in item order.
423fn parallel_owned<T: Send, R: Send>(items: Vec<T>, f: impl Fn(T) -> R + Sync) -> Vec<R> {
424    let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(items.len());
425    if threads <= 1 {
426        return items.into_iter().map(f).collect();
427    }
428    let size = items.len().div_ceil(threads);
429    let mut parts: Vec<Vec<T>> = Vec::with_capacity(threads);
430    let mut rest = items.into_iter();
431    loop {
432        let part: Vec<T> = rest.by_ref().take(size).collect();
433        if part.is_empty() {
434            break;
435        }
436        parts.push(part);
437    }
438    std::thread::scope(|s| {
439        let handles: Vec<_> = parts
440            .into_iter()
441            .map(|part| {
442                let f = &f;
443                s.spawn(move || part.into_iter().map(f).collect::<Vec<R>>())
444            })
445            .collect();
446        handles.into_iter().flat_map(|h| h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect()
447    })
448}
449
450/// The function hash `name` means at the end of time: the file's own over the store's.
451fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
452    if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
453        return Ok(Some(h));
454    }
455    store.resolve_name(name, END).map_err(|e| e.to_string())
456}
457
458/// The class hash `name` means at the end of time: the file's own over the store's.
459fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
460    if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
461        return Ok(Some(h));
462    }
463    store.resolve_class(name, END).map_err(|e| e.to_string())
464}
465
466/// The function hash `name` means as of `at`, counting the file's earlier binds.
467fn function_at<S: Store + ?Sized>(
468    reads: &mut Reads,
469    store: &S,
470    sim: &Simulation,
471    name: &str,
472    at: &str,
473) -> Result<Option<String>, String> {
474    let stored = reads.binding(store, name, at)?;
475    let simmed = Simulation::latest(&sim.functions, name, at);
476    Ok(match (stored, simmed) {
477        (Some(b), Some((t, h))) if t >= b.tx => Some(h),
478        (Some(b), _) => Some(b.fn_hash),
479        (None, Some((_, h))) => Some(h),
480        (None, None) => None,
481    })
482}
483
484/// The latest binding tx of `name` (a class's, or a function's), the store's or the file's.
485fn latest_bind<S: Store + ?Sized>(
486    store: &S,
487    sim: &Simulation,
488    name: &str,
489    class: bool,
490) -> Result<Option<String>, String> {
491    let stored: Option<String> = if class {
492        store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
493    } else {
494        store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
495    };
496    let binds = if class { &sim.classes } else { &sim.functions };
497    let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
498    Ok(match (stored, planned) {
499        (Some(a), Some(b)) => Some(a.max(b)),
500        (a, b) => a.or(b),
501    })
502}
503
504/// Whether `name` is bound as the other kind: a function for a class, a class for a function.
505fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
506    if class {
507        Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
508    } else {
509        Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
510            || sim.classes.contains_key(name))
511    }
512}
513
514/// The latest version of a cell, the store's or the file's: its tx and receipt hash.
515fn cell_latest<S: Store + ?Sized>(
516    reads: &mut Reads,
517    store: &S,
518    sim: &Simulation,
519    id: &str,
520) -> Result<Option<(String, String)>, String> {
521    let stored = reads.latest(store, id)?;
522    let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
523    Ok(match (stored, planned) {
524        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
525        (a, b) => a.or(b),
526    })
527}
528
529/// The receipt hash of a cell's version as of `at`, the store's or the file's.
530fn cell_version_asof<S: Store + ?Sized>(
531    store: &S,
532    sim: &Simulation,
533    id: &str,
534    at: &str,
535) -> Result<Option<String>, String> {
536    let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
537    let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
538    let latest = match (stored, planned) {
539        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
540        (a, b) => a.or(b),
541    };
542    Ok(latest.map(|x| x.1))
543}
544
545/// The descriptor bytes of a function hash: the file's own, else the store's (memoized).
546fn function_bytes_of<S: Store + ?Sized>(
547    reads: &mut Reads,
548    store: &S,
549    sim: &Simulation,
550    hash: &str,
551) -> Result<Option<Vec<u8>>, String> {
552    if let Some(b) = sim.descriptors.get(hash) {
553        return Ok(Some(b.clone()));
554    }
555    Ok(reads.descriptor(store, hash))
556}
557
558/// Whether a namespace is suspended (the store refuses binds and class puts under it).
559fn suspended<S: Store + ?Sized>(reads: &mut Reads, store: &S, host: &str, line: usize) -> Result<(), PublishError> {
560    match reads.namespace(store, host) {
561        Ok(Some(n)) if n.suspended_at.is_some() => {
562            Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
563        }
564        Ok(_) => Ok(()),
565        Err(e) => Err(PublishError::at(line, e)),
566    }
567}
568
569/// A binding a run would make: the line that makes it and its name.
570struct Bind {
571    line: usize,
572    name: String,
573}
574
575/// What a run writes, for the backfill log: the namespaces and starts of its binds.
576#[derive(Default)]
577struct Written {
578    namespaces: BTreeSet<String>,
579    starts: BTreeSet<String>,
580}
581
582impl Written {
583    fn bound(&mut self, name: &str, start: &str) {
584        self.namespaces.insert(namespace_of(name).to_string());
585        self.starts.insert(start.to_string());
586    }
587}
588
589/// The per-run values both passes use.
590struct Ctx<'a> {
591    host: &'a str,
592    owned: &'a [String],
593    /// The start of the mesh's key.
594    key_from: &'a str,
595    /// The clock's now, read once for the run.
596    now: &'a str,
597    /// The directory of the file (blobs are relative to it).
598    base: &'a Path,
599}
600
601/// One write the second pass makes, planned (and checked) by the first.
602enum Op {
603    /// A descriptor line: put; counted when newly put.
604    Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
605    /// A function (stored or formula): put its descriptor, and bind the name when `bind`.
606    Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
607    /// A class: put it, and bind it (its methods with it, in the store) when `bind`.
608    Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
609    /// A cell: its blob, its receipt, and the version when `version`.
610    Cell { tx: String, id: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
611    /// A catalogue line (R5): the entry, and its dictionaries (`publisher`, signed by the run's key).
612    Catalog { entry: datumesh_store::CatalogEntry, dictionaries: Vec<(String, Vec<datumesh_store::KeyValue>)> },
613}
614
615/// The lines of a file, numbered, without the blank ones.
616fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
617    let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
618    let mut out = Vec::new();
619    for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
620        let n = i + 1;
621        let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
622        if !line.trim().is_empty() {
623            out.push((n, line));
624        }
625    }
626    Ok(out)
627}
628
629/// The dictionary values the file's cells have for the string arguments (CONTRACT-build2-part2-read
630/// R3): each distinct (function, key, value) that no dictionary has yet is stored with an empty label
631/// (the value stands for itself). A dictionary's label is never replaced by a cell's.
632fn index_cell_values<S: Store + ?Sized>(store: &S, ops: &[Op]) -> Result<(), String> {
633    let mut seen = BTreeSet::new();
634    for op in ops {
635        if let Op::Cell { receipt, .. } = op {
636            if let Ok(r) = datumesh_core::Receipt::from_json(receipt) {
637                cell_string_values(&r, &mut seen);
638            }
639        }
640    }
641    store_cell_values(store, seen)
642}
643
644/// The (function, key, value) of each string argument of a receipt's cell, into `out`.
645pub(crate) fn cell_string_values(r: &datumesh_core::Receipt, out: &mut BTreeSet<(String, String, String)>) {
646    let cell = r.cell();
647    for (key, arg) in &cell.args {
648        if let datumesh_core::Arg::Value(datumesh_core::Scalar::String(v)) = arg {
649            if !v.is_empty() && v.len() <= 1024 {
650                out.insert((cell.function_name(), key.clone(), v.clone()));
651            }
652        }
653    }
654}
655
656/// Stores the values of [`cell_string_values`] that no dictionary has yet (R3).
657pub(crate) fn store_cell_values<S: Store + ?Sized>(
658    store: &S,
659    seen: BTreeSet<(String, String, String)>,
660) -> Result<(), String> {
661    for (owner, key, value) in seen {
662        if crate::find::has_value(store, &owner, &key, &value) {
663            continue;
664        }
665        let kv = datumesh_store::KeyValue { value, label: String::new(), aliases: Vec::new() };
666        store.upsert_key_values(&owner, &key, std::slice::from_ref(&kv)).map_err(|e| e.to_string())?;
667    }
668    Ok(())
669}
670
671/// Whether a line is a cell line: none of the other kinds (the dispatch order of `plan_line`).
672fn is_cell_line(v: &Value) -> bool {
673    v.get("class").and_then(Value::as_str).is_none()
674        && v.get("descriptor").is_none()
675        && v.get("function").and_then(Value::as_str).is_none()
676        && v.get("formula").and_then(Value::as_str).is_none()
677        && v.get("cell").is_some()
678}
679
680/// A cell line prepared for signing: everything its receipt takes, checked up to there.
681struct Pending {
682    line: usize,
683    ns: String,
684    cell: Cell,
685    value: Scalar,
686    valid: Args,
687    tx: String,
688    asof: String,
689    issued: String,
690    has_tx: bool,
691    licence: String,
692    summary: String,
693    blob: Option<Vec<u8>>,
694}
695
696impl Pending {
697    /// Signs the receipt (deterministic) and takes its hash and cell id. Runs on any thread.
698    fn sign(self, key: &SigningKey, host: &str) -> Result<Signed, PublishError> {
699        let line = self.line;
700        let err = move |d: String| PublishError::at(line, d);
701        let r = ReceiptBuilder::new()
702            .cell(self.cell.clone())
703            .value(self.value)
704            .valid(self.valid)
705            .asof(self.asof.as_str())
706            .tx(self.tx.as_str())
707            .issued(self.issued.as_str())
708            .publisher(self.ns.as_str())
709            .signer_host(host)
710            .licence(self.licence.as_str())
711            .summary(self.summary.as_str())
712            .sign(key)
713            .map_err(|e| err(e.to_string()))?;
714        let receipt = receipt_json(&r);
715        let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
716        let id = self.cell.id();
717        Ok(Signed { line, has_tx: self.has_tx, tx: self.tx, id, hash, receipt, blob: self.blob })
718    }
719}
720
721/// A cell line signed: what its checks and its write take.
722struct Signed {
723    line: usize,
724    has_tx: bool,
725    tx: String,
726    id: String,
727    hash: String,
728    receipt: Vec<u8>,
729    blob: Option<Vec<u8>>,
730}
731
732/// Pass one: plans the writes of the lines and checks each as the store will (CONTRACT-1.7a H2).
733struct Planner<'a, S: Store + ?Sized> {
734    store: &'a S,
735    key: &'a SigningKey,
736    ctx: Ctx<'a>,
737    backfill: bool,
738    sim: Simulation,
739    reads: Reads,
740    ops: Vec<Op>,
741    binds: Vec<Bind>,
742    /// The test lines of the file, with their lines (Build 2 Part 3).
743    tests: Vec<(usize, TestSpec)>,
744    /// The computed functions the file defines: (line, name, function hash).
745    formulas: Vec<(usize, String, String)>,
746}
747
748impl<S: Store + ?Sized> Planner<'_, S> {
749    fn callers(&self) -> Callers<'_, S> {
750        Callers { store: self.store, sim: Some(&self.sim) }
751    }
752
753    /// Plans the lines in file order. Cell lines are planned in chunks of up to `CHUNK`
754    /// consecutive ones; any other line ends a chunk (CONTRACT-perf H1).
755    fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
756        let mut cells: Vec<(usize, Value)> = Vec::new();
757        for (n, text) in lines {
758            let n = *n;
759            match json::parse(text.as_bytes()).map_err(|e| PublishError::at(n, e.to_string())) {
760                Ok(v) if is_cell_line(&v) => {
761                    cells.push((n, v));
762                    if cells.len() == CHUNK {
763                        self.plan_cells(std::mem::take(&mut cells))?;
764                    }
765                }
766                other => {
767                    // The cells before this line are checked first: a failure of either is the
768                    // file's first.
769                    self.plan_cells(std::mem::take(&mut cells))?;
770                    self.plan_line(n, &other?)?;
771                }
772            }
773        }
774        self.plan_cells(cells)
775    }
776
777    /// One line that is not a cell (the dispatch of the lines before cells, as it was).
778    fn plan_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
779        let err = |d: String| PublishError::at(n, d);
780        if let Some(name) = v.get("class").and_then(Value::as_str) {
781            self.class_line(n, name, v)?;
782        } else if let Some(d) = v.get("descriptor") {
783            self.descriptor_line(n, d)?;
784        } else if let Some(name) = v.get("function").and_then(Value::as_str) {
785            let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
786            let start = bound_of(v, n, self.ctx.now)?;
787            let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
788            self.function(n, name, start, fd.hash(), fd.canonical())?;
789        } else if v.get("catalog").is_some() {
790            self.catalog_line(n, v)?;
791        } else if let Some(name) = v.get("formula").and_then(Value::as_str) {
792            let source =
793                v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
794            let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
795            let start = bound_of(v, n, self.ctx.now)?;
796            let def = datumesh_formula::definition::define_function(
797                name,
798                source,
799                sig,
800                &Default::default(),
801                &self.callers(),
802                &start,
803            )
804            .map_err(|e| err(format!("define {name}: {e}")))?;
805            let hash = def.fn_hash.clone();
806            self.function(n, name, start, def.fn_hash, def.bytes)?;
807            self.formulas.push((n, name.to_string(), hash));
808        } else if v.get("test").is_some() {
809            let spec = testrun::parse_test(v).map_err(err)?;
810            self.tests.push((n, spec));
811        } else {
812            return Err(err("neither a function nor a cell".into()));
813        }
814        Ok(())
815    }
816
817    /// `{"catalog": <name>, "kind", "label", "description", "args", "examples", "dictionary"}` (R5):
818    /// the publisher's text for a function or class bound in this file or the store, under a host
819    /// this mesh owns. Checked here, so a refused line writes nothing.
820    fn catalog_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
821        let err = |d: String| PublishError::at(n, d);
822        let d = crate::catalog::parse(v, "catalog").map_err(err)?;
823        let host = host_of(&d.name);
824        if host != self.ctx.host && !self.ctx.owned.iter().any(|o| o == host) {
825            return Err(err(format!("{} is not under a host this mesh owns", d.name)));
826        }
827        let hash = match d.kind {
828            datumesh_store::CatalogKind::Class => current_class(self.store, &self.sim, &d.name).map_err(err)?,
829            _ => current_function(self.store, &self.sim, &d.name).map_err(err)?,
830        };
831        if hash.is_none() {
832            return Err(err(format!("{} is bound nowhere", d.name)));
833        }
834        let entry = d.entry(hash, datumesh_store::CatalogSource::Publisher, Some(self.key.key_id().to_string()));
835        self.ops.push(Op::Catalog { entry, dictionaries: d.dictionary });
836        Ok(())
837    }
838
839    /// `{"descriptor": ...}`: a stored function's descriptor, put and bound to nothing.
840    fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
841        let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
842        if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
843            return Err(PublishError::at(
844                n,
845                "a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
846            ));
847        }
848        let (hash, bytes) = (fd.hash(), fd.canonical());
849        self.sim.descriptors.insert(hash.clone(), bytes.clone());
850        self.ops.push(Op::Descriptor { hash, bytes, counted: true });
851        Ok(())
852    }
853
854    /// A function's descriptor (stored or formula): put, and bound when its name does not mean it.
855    fn function(
856        &mut self,
857        n: usize,
858        name: &str,
859        start: String,
860        hash: String,
861        bytes: Vec<u8>,
862    ) -> Result<(), PublishError> {
863        self.sim.descriptors.insert(hash.clone(), bytes.clone());
864        let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
865        let bind = current.as_deref() != Some(hash.as_str());
866        if bind {
867            self.check_bind(n, name, &start, false)?;
868            self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
869        }
870        self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
871        Ok(())
872    }
873
874    fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
875        let err = |d: String| PublishError::at(n, d);
876        let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
877        let start = bound_of(v, n, self.ctx.now)?;
878        let def = datumesh_formula::definition::define_class(
879            name,
880            &descriptor.to_jcs(),
881            definer(name, self.ctx.host, self.ctx.owned),
882            &self.callers(),
883            &start,
884        )
885        .map_err(|e| err(format!("define {name}: {e}")))?;
886        // put_class refuses under a suspended steward namespace, whether or not the name binds. A
887        // namespace row is a host's (9.9), so the steward namespace's host is asked: a steward with
888        // a path (datumesh.com/scripture) is not a host and Store::namespace refuses it.
889        suspended(&mut self.reads, self.store, host_of(&def.descriptor.steward.namespace), n)?;
890        let hash = def.class_hash.clone();
891        let methods: Vec<(String, String)> =
892            def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
893        let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
894        if bind {
895            self.check_bind(n, name, &start, true)?;
896            // bind_class binds every method in the same transaction (the store's rule).
897            for (mname, _) in &methods {
898                self.check_bind(n, mname, &start, false)?;
899            }
900            self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
901            for (mname, fh) in &methods {
902                self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
903            }
904        }
905        self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
906        self.ops.push(Op::Class {
907            name: name.to_string(),
908            start,
909            hash,
910            bytes: def.bytes,
911            bind,
912            methods: methods.into_iter().map(|(m, _)| m).collect(),
913        });
914        Ok(())
915    }
916
917    /// One bind a line would make: rule A, the other kind, the namespace, and the latest binding.
918    fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
919        if !self.backfill && start < self.ctx.now {
920            return Err(PublishError::typed(
921                line,
922                PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
923            ));
924        }
925        let err = |d: String| PublishError::at(line, d);
926        if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
927            let kind = if class { "function" } else { "class" };
928            return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
929        }
930        suspended(&mut self.reads, self.store, host_of(name), line)?;
931        if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
932            if start <= latest.as_str() {
933                return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
934            }
935        }
936        self.binds.push(Bind { line, name: name.into() });
937        Ok(())
938    }
939
940    /// Plans the cell lines of a chunk (CONTRACT-perf H1): each is prepared in file order up to its
941    /// receipt; the receipts are signed on threads; the store is read once for the chunk; then each
942    /// is checked in file order. The first failure is the chunk's, and a line that fails before
943    /// signing is reported after the lines before it are checked.
944    fn plan_cells(&mut self, lines: Vec<(usize, Value)>) -> Result<(), PublishError> {
945        if lines.is_empty() {
946            return Ok(());
947        }
948        let mut pending = Vec::with_capacity(lines.len());
949        let mut failed = None;
950        for (n, v) in &lines {
951            match self.prepare_cell(*n, v) {
952                Ok(p) => pending.push(p),
953                Err(e) => {
954                    failed = Some(e);
955                    break;
956                }
957            }
958        }
959        let (key, host) = (self.key, self.ctx.host);
960        let signed = parallel_owned(pending, |p| p.sign(key, host));
961        let ids: Vec<&str> = signed.iter().flatten().map(|s| s.id.as_str()).collect();
962        self.reads.prefetch_latest(self.store, &ids);
963        let hashes: Vec<String> = signed
964            .iter()
965            .flatten()
966            .filter(|s| !s.has_tx)
967            .filter_map(|s| self.reads.latest_of.get(&s.id).cloned().flatten())
968            .map(|(_, h)| h)
969            .collect();
970        let hashes: Vec<&str> = hashes.iter().map(String::as_str).collect();
971        self.reads.prefetch_receipts(self.store, &hashes);
972        for s in signed {
973            self.check_cell(s?)?;
974        }
975        match failed {
976            Some(e) => Err(e),
977            None => Ok(()),
978        }
979    }
980
981    /// A cell line up to its receipt, as the checks of a line take them in order: the fields, the
982    /// instants, the function as of `tx`, the descriptor, the arguments and the value (and its blob).
983    fn prepare_cell(&mut self, n: usize, v: &Value) -> Result<Pending, PublishError> {
984        let err = |d: String| PublishError::at(n, d);
985        let cell_v = v.get("cell").ok_or_else(|| err("a cell without fn".into()))?;
986        let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
987        let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
988        let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
989        let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
990        let (licence, summary) = (required("licence")?, required("summary")?);
991        // tx and asof default to the run's clock now, each on its own (H4).
992        let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
993        let has_tx = field("tx").is_some();
994        let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
995        if tx > asof {
996            return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
997        }
998        let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
999        if asof > issued {
1000            return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
1001        }
1002        let fn_hash = function_at(&mut self.reads, self.store, &self.sim, name, &tx)
1003            .map_err(err)?
1004            .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
1005        let d = function_bytes_of(&mut self.reads, self.store, &self.sim, &fn_hash)
1006            .map_err(err)?
1007            .ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
1008        let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
1009        let mut args = Args::new();
1010        for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
1011            args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
1012        }
1013        let valid: Args = args
1014            .iter()
1015            .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
1016            .map(|(k, a)| (k.clone(), a.clone()))
1017            .collect();
1018        let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
1019        let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
1020            Some(file) => {
1021                let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
1022                let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
1023                let sha = sha256_hex(&bytes);
1024                let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
1025                    .map_err(|e| err(e.to_string()))?;
1026                (scalar, Some(bytes))
1027            }
1028            None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
1029        };
1030        let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
1031        Ok(Pending {
1032            line: n,
1033            ns: ns.to_string(),
1034            cell,
1035            value,
1036            valid,
1037            tx,
1038            asof,
1039            issued,
1040            has_tx,
1041            licence,
1042            summary,
1043            blob,
1044        })
1045    }
1046
1047    /// The checks of a signed cell line, in file order: the skip rule of a line without `tx`, then
1048    /// the version rule. Its write is planned.
1049    fn check_cell(&mut self, s: Signed) -> Result<(), PublishError> {
1050        let line = s.line;
1051        let err = |d: String| PublishError::at(line, d);
1052        // A line without tx takes the run's clock, so a re-run at a later clock would sign the same
1053        // fact again as a new version. Re-running a file changes nothing: when the cell's latest
1054        // version is the store's and states the same thing (type, value, valid, licence, summary),
1055        // the line is skipped. A changed value is a new version, a correction.
1056        if !s.has_tx {
1057            if let Some((_, latest)) = cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1058                if let Some(old) = self.reads.receipt(self.store, &latest) {
1059                    let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
1060                    let new = json::parse(&s.receipt).map_err(|e| err(e.to_string()))?;
1061                    if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
1062                        return Ok(());
1063                    }
1064                }
1065            }
1066        }
1067        let version = match cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1068            Some((latest_tx, _)) if latest_tx.as_str() >= s.tx.as_str() => {
1069                if cell_version_asof(self.store, &self.sim, &s.id, &s.tx).map_err(err)?.as_deref()
1070                    != Some(s.hash.as_str())
1071                {
1072                    return Err(err(format!("{} has a version at or after {} that is not this receipt", s.id, s.tx)));
1073                }
1074                false
1075            }
1076            _ => {
1077                self.sim.versions.entry(s.id.clone()).or_default().push((s.tx.clone(), s.hash.clone()));
1078                true
1079            }
1080        };
1081        self.ops.push(Op::Cell { tx: s.tx, id: s.id, receipt: s.receipt, blob: s.blob, version });
1082        Ok(())
1083    }
1084}
1085
1086/// The stored descriptor of a function line's signature.
1087fn stored_value(signature: &Value) -> Value {
1088    Value::Object(vec![
1089        ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
1090        ("inputs".into(), Value::Array(vec![])),
1091        ("signature".into(), signature.clone()),
1092    ])
1093}
1094
1095/// The backfill's checks (since 0.3.0 with store 0.4.2): no name the file binds has a binding, and
1096/// no namespace of one has any binding (`has_bindings_under`).
1097fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
1098    for b in binds {
1099        let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
1100        if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
1101            return Err(refuse(format!("{} has a binding", b.name)));
1102        }
1103        let ns = namespace_of(&b.name);
1104        if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
1105            return Err(refuse(format!("{ns} has bindings")));
1106        }
1107    }
1108    Ok(())
1109}
1110
1111/// Appends a run's line to the backfill log (created with mode 0600).
1112fn append_log(log: &Path, written: &Written) -> Result<(), String> {
1113    let line = Value::Object(vec![
1114        ("at".into(), s(&SystemClock.now())),
1115        ("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
1116        ("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
1117    ])
1118    .to_jcs();
1119    let mut opts = std::fs::OpenOptions::new();
1120    opts.create(true).append(true);
1121    #[cfg(unix)]
1122    {
1123        use std::os::unix::fs::OpenOptionsExt;
1124        opts.mode(0o600);
1125    }
1126    let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
1127    let mut bytes = line;
1128    bytes.push(b'\n');
1129    f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
1130}
1131
1132/// Pass two: the writes the first pass planned, in order. Nothing here can refuse a planned
1133/// write, so a refused file has written nothing. Consecutive cell writes go in `put_cells` calls
1134/// of up to `CHUNK` on `lanes` lanes, their blobs put first (CONTRACT-perf H2, H5).
1135fn write<S: Store + ?Sized>(
1136    store: &S,
1137    ops: &[Op],
1138    cells: impl Fn(&S, &[Op], &mut Published) -> Result<(), String>,
1139) -> Result<(Published, Written), PublishError> {
1140    let mut out = Published::default();
1141    let mut written = Written::default();
1142    let err = |d: String| PublishError::at(0, d);
1143    let mut i = 0;
1144    while i < ops.len() {
1145        match &ops[i] {
1146            Op::Descriptor { hash, bytes, counted } => {
1147                let existed = store.descriptor(hash).is_some();
1148                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1149                if *counted && !existed {
1150                    out.descriptors += 1;
1151                }
1152            }
1153            Op::Function { name, start, hash, bytes, bind } => {
1154                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1155                if *bind {
1156                    store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1157                    out.functions += 1;
1158                    written.bound(name, start);
1159                }
1160            }
1161            Op::Class { name, start, hash, bytes, bind, methods } => {
1162                store.put_class(bytes).map_err(|e| err(e.to_string()))?;
1163                if *bind {
1164                    store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1165                    out.classes += 1;
1166                    written.bound(name, start);
1167                    for m in methods {
1168                        written.bound(m, start);
1169                    }
1170                }
1171            }
1172            Op::Catalog { entry, dictionaries } => {
1173                crate::catalog::store_publisher(store, entry, dictionaries).map_err(err)?;
1174            }
1175            Op::Cell { .. } => {
1176                let mut j = i;
1177                while j < ops.len() && j - i < CHUNK && matches!(ops[j], Op::Cell { .. }) {
1178                    j += 1;
1179                }
1180                cells(store, &ops[i..j], &mut out).map_err(err)?;
1181                i = j;
1182                continue;
1183            }
1184        }
1185        i += 1;
1186    }
1187    Ok((out, written))
1188}
1189
1190/// A run of cell writes (up to `CHUNK`) on one lane: their blobs first, then the receipts and
1191/// versions in file order in `put_cells` calls of up to `CHUNK` (CONTRACT-perf H2). No `Sync`
1192/// bound: this is what [`publish_stored_with`] runs.
1193fn write_cells_one_lane<S: Store + ?Sized>(store: &S, group: &[Op], out: &mut Published) -> Result<(), String> {
1194    let items = put_blobs_and_items(store, group, out)?;
1195    let failed = AtomicBool::new(false);
1196    let ks: Vec<usize> = (0..items.len()).collect();
1197    if let Some((_, detail)) = put_lane(store, &items, &ks, &failed) {
1198        return Err(detail);
1199    }
1200    count_cells(group, &items, out);
1201    Ok(())
1202}
1203
1204/// A run of cell writes (up to `CHUNK`): their blobs first, then their receipts and versions on
1205/// `lanes` lanes (CONTRACT-perf H5). A cell's id picks its lane (its first hex digit, mod `lanes`),
1206/// so all writes of one cell share a lane, in file order. A lane's `put_cells` calls take up to
1207/// `CHUNK` of its items. When one fails, no lane starts another call, and the error returned is
1208/// that of the earliest failing item in file order. With one lane, as [`write_cells_one_lane`].
1209fn write_cells<S: Store + Sync + ?Sized>(
1210    store: &S,
1211    group: &[Op],
1212    lanes: usize,
1213    out: &mut Published,
1214) -> Result<(), String> {
1215    if lanes <= 1 {
1216        return write_cells_one_lane(store, group, out);
1217    }
1218    let items = put_blobs_and_items(store, group, out)?;
1219    let mut by_lane: Vec<Vec<usize>> = vec![Vec::new(); lanes];
1220    for (k, op) in group.iter().enumerate() {
1221        if let Op::Cell { id, .. } = op {
1222            by_lane[lane_of(id, lanes)].push(k);
1223        }
1224    }
1225    let failed = AtomicBool::new(false);
1226    let failures: Vec<(usize, String)> = std::thread::scope(|scope| {
1227        let handles: Vec<_> = by_lane
1228            .iter()
1229            .map(|ks| {
1230                let (items, failed) = (&items, &failed);
1231                scope.spawn(move || put_lane(store, items, ks, failed))
1232            })
1233            .collect();
1234        handles.into_iter().filter_map(|h| h.join().unwrap_or_else(|p| std::panic::resume_unwind(p))).collect()
1235    });
1236    if let Some((_, detail)) = failures.into_iter().min_by_key(|(k, _)| *k) {
1237        return Err(detail);
1238    }
1239    count_cells(group, &items, out);
1240    Ok(())
1241}
1242
1243/// The blobs of a run, put in order (counted in `out`), and the run's cell writes.
1244fn put_blobs_and_items<'a, S: Store + ?Sized>(
1245    store: &S,
1246    group: &'a [Op],
1247    out: &mut Published,
1248) -> Result<Vec<CellItem<'a>>, String> {
1249    for op in group {
1250        if let Op::Cell { blob: Some(bytes), .. } = op {
1251            store.put_blob(bytes).map_err(|e| e.to_string())?;
1252            out.blobs += 1;
1253            out.blob_bytes += bytes.len() as u64;
1254        }
1255    }
1256    Ok(group
1257        .iter()
1258        .filter_map(|op| match op {
1259            Op::Cell { tx, id, receipt, version, .. } => {
1260                Some((receipt.as_slice(), if *version { Some((id.as_str(), tx.as_str())) } else { None }))
1261            }
1262            _ => None,
1263        })
1264        .collect())
1265}
1266
1267/// Counts a written run of cells in `out`.
1268fn count_cells(group: &[Op], items: &[CellItem<'_>], out: &mut Published) {
1269    out.cells += group.len();
1270    out.receipts += group.len();
1271    out.versions += items.iter().filter(|(_, v)| v.is_some()).count();
1272}
1273
1274/// The lane of a cell id: its first hex digit, mod `lanes` (CONTRACT-perf H5).
1275fn lane_of(id: &str, lanes: usize) -> usize {
1276    id.bytes().next().and_then(|b| (b as char).to_digit(16)).unwrap_or(0) as usize % lanes
1277}
1278
1279/// A cell write of a run: its receipt's bytes and its version (cell id, tx), if any.
1280type CellItem<'a> = (&'a [u8], Option<(&'a str, &'a str)>);
1281
1282/// One lane's writes: its items (indices into `items`, in file order) in `put_cells` calls of up to
1283/// `CHUNK`. Returns the index of the first item of the call that failed, and its error. No call
1284/// starts once `failed` is set.
1285fn put_lane<S: Store + ?Sized>(
1286    store: &S,
1287    items: &[CellItem<'_>],
1288    ks: &[usize],
1289    failed: &AtomicBool,
1290) -> Option<(usize, String)> {
1291    for call in ks.chunks(CHUNK) {
1292        if failed.load(Ordering::SeqCst) {
1293            return None;
1294        }
1295        let writes: Vec<CellWrite<'_>> = call.iter().map(|&k| CellWrite::new(items[k].0, items[k].1)).collect();
1296        if let Err(e) = store.put_cells(&writes) {
1297            failed.store(true, Ordering::SeqCst);
1298            return Some((call[0], e.to_string()));
1299        }
1300    }
1301    None
1302}
1303
1304/// The receipt JSON of a signed receipt: its payload's members and `sig`.
1305pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
1306    let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
1307    if let Ok(Value::Object(m)) = json::parse(r.payload()) {
1308        members.extend(m);
1309    }
1310    Value::Object(members).to_jcs()
1311}
1312
1313/// Publishes the file at `path` into `store` as the mesh `host` signing with `key` (see the
1314/// module documentation). The mesh's own namespaces are `host` alone: see [`publish_stored_in`].
1315/// The clock is the system clock, and there is no backfill. The cells are written on one lane;
1316/// see [`publish_stored_lanes`] for the parallel lanes.
1317pub fn publish_stored<S: Store + ?Sized>(
1318    store: &S,
1319    key: &SigningKey,
1320    host: &str,
1321    path: &Path,
1322) -> Result<Published, PublishError> {
1323    publish_stored_in(store, key, host, &[host.to_string()], path)
1324}
1325
1326/// As [`publish_stored`], for a mesh that owns the namespaces (hosts) `owned` (`Config::namespaces`):
1327/// a class line is accepted for a class under one of them, and refused with `E_CLASS_OWNER` (in the
1328/// [`PublishError::detail`]) under any other. Since 0.2.2. The system clock, and no backfill.
1329pub fn publish_stored_in<S: Store + ?Sized>(
1330    store: &S,
1331    key: &SigningKey,
1332    host: &str,
1333    owned: &[String],
1334    path: &Path,
1335) -> Result<Published, PublishError> {
1336    publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
1337}
1338
1339/// As [`publish_stored_in`], with the run's clock and backfill (CONTRACT-1.8b H1, rule A). The
1340/// cells are written in file order on one lane, as in 0.3.2 (CONTRACT-perf H2).
1341///
1342/// Errors: a line refused by its checks, or a binding whose start is earlier than the clock
1343/// ([`PublishErrorKind::StartBeforeNow`]), or a backfill refused ([`PublishErrorKind::Backfill`]).
1344/// Nothing is written when the first pass refuses (CONTRACT-1.7a H2).
1345pub fn publish_stored_with<S: Store + ?Sized>(
1346    store: &S,
1347    key: &SigningKey,
1348    host: &str,
1349    owned: &[String],
1350    path: &Path,
1351    opts: &PublishOptions,
1352) -> Result<Published, PublishError> {
1353    publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1354        write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1355    })
1356}
1357
1358/// As [`publish_stored_with`], with the public rule (Build 2 Part 3, D8: datumesh-http always
1359/// enforces it; `publish-stored` calls this). A file that defines a computed function (formula)
1360/// is refused unless the tests of the function (the file's, and those already held for the same
1361/// function hash) exercise every derived edge case and every branch, and all of the file's tests
1362/// pass: [`PublishErrorKind::TestsMissing`] at the function's line, [`PublishErrorKind::TestsFailing`]
1363/// at the failing test's line. Stored functions are exempt. Nothing is written when refused.
1364pub fn publish_stored_checked<S: Store + ?Sized>(
1365    store: &S,
1366    key: &SigningKey,
1367    host: &str,
1368    owned: &[String],
1369    path: &Path,
1370    opts: &PublishOptions,
1371) -> Result<Published, PublishError> {
1372    publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1373        write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1374    })
1375}
1376
1377/// As [`publish_stored_with`], with the cells written on parallel lanes (CONTRACT-perf H5, H6):
1378/// `DATUMESH_PUBLISH_LANES` (a positive integer) if set, else `min(4, available_parallelism())`.
1379/// The same receipts, versions and rows as one lane; only the order of the inserts across
1380/// different cells differs. The variable is read once per run, before the file is read.
1381pub fn publish_stored_lanes<S: Store + Sync + ?Sized>(
1382    store: &S,
1383    key: &SigningKey,
1384    host: &str,
1385    owned: &[String],
1386    path: &Path,
1387    opts: &PublishOptions,
1388) -> Result<Published, PublishError> {
1389    let lanes = publish_lanes()?;
1390    publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1391        write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1392    })
1393}
1394
1395/// The planned writes of a run as the tests see them (see [`Overlay`]).
1396fn overlay_of(ops: &[Op]) -> Result<Overlay, PublishError> {
1397    let mut overlay = Overlay::default();
1398    for op in ops {
1399        match op {
1400            Op::Function { name, start, hash, bytes, bind } => {
1401                overlay.descriptors.insert(hash.clone(), bytes.clone());
1402                if *bind {
1403                    overlay.functions.entry(name.clone()).or_default().push((start.clone(), hash.clone()));
1404                }
1405            }
1406            Op::Descriptor { hash, bytes, .. } => {
1407                overlay.descriptors.insert(hash.clone(), bytes.clone());
1408            }
1409            Op::Cell { receipt, .. } => {
1410                overlay.cell(receipt).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1411            }
1412            Op::Class { .. } | Op::Catalog { .. } => {}
1413        }
1414    }
1415    Ok(overlay)
1416}
1417
1418/// The public rule over a file (Build 2 Part 3, E5): runs the tests of each computed function the
1419/// file defines, over the store and the file's planned writes, and refuses the file when `checked`
1420/// and a test fails or the tests leave a derived case or a branch arm unexercised. Returns the
1421/// rows of the tests to hold (the file's and the held ones of those functions).
1422#[allow(clippy::too_many_arguments)]
1423pub(crate) fn check_tests<S: Store + ?Sized>(
1424    store: &S,
1425    overlay: &Overlay,
1426    tests: &[(usize, TestSpec)],
1427    formulas: &[(usize, String, String)],
1428    host: &str,
1429    now: &str,
1430    checked: bool,
1431) -> Result<Vec<RowWrite>, PublishError> {
1432    if formulas.is_empty() {
1433        return Ok(Vec::new());
1434    }
1435    let mut file_tests: Vec<(usize, TestSpec, String)> = Vec::new();
1436    for (n, spec) in tests {
1437        let at = spec.asof.clone().unwrap_or_else(|| now.to_string());
1438        match testrun::binding_of(store, overlay, &spec.function, &at) {
1439            Ok(Some((hash, _))) => file_tests.push((*n, spec.clone(), hash)),
1440            Ok(None) => {
1441                return Err(PublishError::at(
1442                    *n,
1443                    format!("a test of {}: no function of that name is bound", spec.function),
1444                ))
1445            }
1446            Err(e) => return Err(PublishError::at(*n, e)),
1447        }
1448    }
1449    let mut rows = Vec::new();
1450    for (line, name, hash) in formulas {
1451        let own: Vec<(usize, TestSpec)> =
1452            file_tests.iter().filter(|t| &t.2 == hash).map(|t| (t.0, t.1.clone())).collect();
1453        let held = testrun::held_tests(store, hash).map_err(|e| PublishError::at(*line, e))?;
1454        if !checked && own.is_empty() && held.is_empty() {
1455            continue;
1456        }
1457        let mut specs: Vec<(Option<usize>, TestSpec)> = own.into_iter().map(|(n, s)| (Some(n), s)).collect();
1458        for (_, spec) in held {
1459            if !specs.iter().any(|(_, s)| s.case == spec.case) {
1460                specs.push((None, spec));
1461            }
1462        }
1463        let ran: Vec<Ran> = specs.iter().map(|(_, s)| run_test(store, overlay, now, s)).collect();
1464        let mut run = testrun::Run::new(store, overlay, now);
1465        let built = run.built(name, hash).map_err(|e| PublishError::at(*line, format!("{name}: {}", e.0)))?;
1466        let Some(compiled) = built.compiled.as_ref() else { continue };
1467        if checked {
1468            let mut failing: Vec<(usize, String)> = Vec::new();
1469            for ((n, spec), r) in specs.iter().zip(&ran) {
1470                if let Some(n) = n {
1471                    let (result, detail) = testrun::result_of(spec, r);
1472                    if result != "pass" {
1473                        failing.push((*n, format!("{name}: test {} {detail}", spec.case)));
1474                    }
1475                }
1476            }
1477            failing.sort_by_key(|f| f.0);
1478            if let Some((n, detail)) = failing.into_iter().next() {
1479                return Err(PublishError::typed(n, PublishErrorKind::TestsFailing(detail)));
1480            }
1481            let refs: Vec<&Ran> = ran.iter().collect();
1482            let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1483            if !cov.uncovered.is_empty() || !cov.unexercised.is_empty() {
1484                let mut text = format!("{name} is missing tests:");
1485                if !cov.uncovered.is_empty() {
1486                    let cases: Vec<&str> = cov.uncovered.iter().map(|c| c.0.as_str()).collect();
1487                    text += &format!(" uncovered cases {};", cases.join(", "));
1488                }
1489                if !cov.unexercised.is_empty() {
1490                    text += &format!(" unexercised branches {}", cov.unexercised.join(", "));
1491                }
1492                return Err(PublishError::typed(
1493                    *line,
1494                    PublishErrorKind::TestsMissing(text.trim_end_matches(';').to_string()),
1495                ));
1496            }
1497        }
1498        let refs: Vec<&Ran> = ran.iter().collect();
1499        let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1500        for ((_, spec), r) in specs.iter().zip(&ran) {
1501            let (result, detail) = testrun::result_of(spec, r);
1502            rows.push(RowWrite {
1503                fn_hash: hash.clone(),
1504                case: spec.case.clone(),
1505                jcs: spec.jcs.clone(),
1506                signed_by: host.to_string(),
1507                result,
1508                detail,
1509                branches_total: cov.branches,
1510                branches_covered: cov.covered,
1511            });
1512        }
1513    }
1514    Ok(rows)
1515}
1516
1517/// As [`publish_stored_lanes`], with the public rule (see [`publish_stored_checked`]): the path of
1518/// `datumesh-mesh publish-stored` (D8).
1519pub fn publish_stored_checked_lanes<S: Store + Sync + ?Sized>(
1520    store: &S,
1521    key: &SigningKey,
1522    host: &str,
1523    owned: &[String],
1524    path: &Path,
1525    opts: &PublishOptions,
1526) -> Result<Published, PublishError> {
1527    let lanes = publish_lanes()?;
1528    publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1529        write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1530    })
1531}
1532
1533/// The first pass (the planner) and the second pass (`write_ops`) of a publish.
1534#[allow(clippy::too_many_arguments)]
1535fn publish_planned<S: Store + ?Sized>(
1536    store: &S,
1537    key: &SigningKey,
1538    host: &str,
1539    owned: &[String],
1540    path: &Path,
1541    opts: &PublishOptions,
1542    checked: bool,
1543    write_ops: impl FnOnce(&S, &[Op]) -> Result<(Published, Written), PublishError>,
1544) -> Result<Published, PublishError> {
1545    let lines = read_lines(path)?;
1546    // The key's start: the earliest `issued`.
1547    let key_from = store
1548        .namespace(host)
1549        .ok()
1550        .flatten()
1551        .and_then(|n| json::parse(&n.keydoc).ok())
1552        .and_then(|d| match d.get("keys") {
1553            Some(Value::Array(ks)) => ks
1554                .iter()
1555                .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
1556                .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
1557            _ => None,
1558        })
1559        .ok_or_else(|| {
1560            PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
1561        })?;
1562    let now = opts.clock.now();
1563    let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
1564    let mut planner = Planner {
1565        store,
1566        key,
1567        ctx,
1568        backfill: opts.backfill.is_some(),
1569        sim: Simulation::default(),
1570        reads: Reads::default(),
1571        ops: Vec::new(),
1572        binds: Vec::new(),
1573        tests: Vec::new(),
1574        formulas: Vec::new(),
1575    };
1576    planner.run(&lines)?;
1577    let Planner { ops, binds, tests, formulas, .. } = planner;
1578    if opts.backfill.is_some() {
1579        backfill_check(store, &binds)?;
1580    }
1581    // The tests of the file's computed functions, run over the store and the file's own writes:
1582    // refused before anything is written (Build 2 Part 3, D8).
1583    let rows = check_tests(store, &overlay_of(&ops)?, &tests, &formulas, host, &now, checked)?;
1584    let (out, written) = write_ops(store, &ops)?;
1585    index_cell_values(store, &ops).map_err(|e| PublishError::at(0, format!("catalogue: {e}")))?;
1586    testrun::apply_rows(store, &rows, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1587    let rebound: Vec<String> = ops
1588        .iter()
1589        .filter_map(|op| match op {
1590            Op::Function { name, bind: true, .. } => Some(name.clone()),
1591            _ => None,
1592        })
1593        .collect();
1594    testrun::rerun_readers(store, &rebound, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1595    if let Some(log) = &opts.backfill {
1596        if out.functions + out.classes + out.versions > 0 {
1597            append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
1598        }
1599    }
1600    Ok(out)
1601}