Skip to main content

datumesh_http/
publish.rs

1//! Publishing stored cells as the mesh (`datumesh-mesh publish-stored`): a JSON lines file of
2//! stored functions and their cells, signed with the mesh's key, put in the store.
3//!
4//! Each line is one of:
5//!
6//! * a descriptor (since 0.3.1, CONTRACT-1.7a H1): `{"descriptor": <function descriptor>}`: the
7//!   descriptor is put with `put_descriptor` and binds nothing. Only a stored function is
8//!   accepted (`code.lang` = `stored`, `code.body` = `""`, `inputs` = `[]`); any other is refused
9//!   at its line. A class later in the file (or in a later run) may name its hash as a method;
10//! * a function: `{"function": "<ns>/<name>", "signature": {...}, "bound": "<instant>"}`: the
11//!   descriptor `{"code": {"body": "", "lang": "stored"}, "inputs": [], "signature": ...}`
12//!   is put and the name bound to it from `bound` (unless it already is);
13//! * a formula function: `{"formula": "<ns>/<name>", "source": "<formula>", "signature":
14//!   {...}, "bound": "<instant>"}`: defined with `datumesh_formula::definition::define_function`
15//!   (its callees as bound at `bound`) and published (`put_descriptor`, `bind_name` from
16//!   `bound`) unless the name is already bound to that function; its cells are evaluated on
17//!   demand by the mesh;
18//! * a class (since 0.2.2, spec-0.5.0, 16.2): `{"class": "<ns>/<name>", "descriptor": {...as
19//!   written...}, "bound": "<instant>"}`: defined with the formula crate's `define_class` (core's
20//!   rules: the class is under one of the mesh's own namespaces, else `E_CLASS_OWNER`; the method
21//!   functions are those bound as of `bound`), `put_class`, and `bind_class` from `bound` unless the
22//!   name is already bound to that class hash. Method functions come first in the file (or in an
23//!   earlier one), `bound` later than theirs. A line with both `class` and `descriptor` is a class
24//!   line;
25//! * a cell: `{"cell": {"fn": "<ns>/<name>", "args": {<name>: <typed scalar>}}, "value": <typed
26//!   scalar> or {"blob": "<file>", "media_type": "<type>"}, "tx"?, "asof"?, "issued"?, "licence",
27//!   "summary"}`: a blob's file (relative to the JSON lines file) is put with `put_blob` and the
28//!   value is `<sha256>:<length>:<media type>`; the receipt is signed with the mesh's key
29//!   (`computed_by` the mesh host, `publisher` the cell's namespace, `valid` its valid-time
30//!   arguments; `issued` defaults to the later of `asof` and the key's start) and stored with
31//!   `put_receipt` and `put_version`. `tx` and `asof` default to the run's clock now (since
32//!   0.3.1, H4), each independently. A `source` object (since 0.4.0, spec-0.6.0 C2: `name`, `url`,
33//!   `retrieved`, and optionally `sha256` and `takeover`, nothing else) marks the value as relayed: the
34//!   receipt signs it, and its trust is `relayed`; its rules are the receipt's (`retrieved` no later
35//!   than `issued`, a publisher that does not relay itself, `https://` URLs), refused at the line.
36//!
37//! The function of a cell is the one its name is bound to as of the cell's `tx`. Re-running a
38//! file changes nothing (a cell line without `tx` whose latest version states the same thing is
39//! skipped, since 0.3.1): descriptors and blobs are content-addressed, signing is deterministic
40//! (the same receipt, the same hash), and a version already there is skipped. The key is never
41//! logged.
42//!
43//! Rule A (since 0.3.0, CONTRACT-1.8b H1): no binding starts earlier than the mesh clock. The
44//! start of a binding (a name's, a class's, or a class method's `N/m`) is its `bound`, or the
45//! clock's now when `bound` is left out, and a start earlier than now is refused with
46//! [`PublishErrorKind::StartBeforeNow`]. A `--backfill` run ([`PublishOptions::backfill`]) lifts
47//! the start check, and is refused ([`PublishErrorKind::Backfill`]) when a name it would bind has
48//! a binding, or when a namespace of such a name has any binding (store 0.4.2's
49//! `has_bindings_under`). A run that writes appends one line to the log: `{"at": …, "namespaces":
50//! […], "starts": […]}`, a JCS object.
51//!
52//! No partial writes (since 0.3.1, CONTRACT-1.7a H2): the file is read in two passes. The first
53//! plans every write the second one makes, and checks each as the store will: a bind is refused
54//! when its start is not later than the latest binding of its name (the store's, or one planned
55//! earlier in the file, W1b), when the name is of the other kind (a class or a function), or when
56//! its namespace is suspended; a cell's function must be bound as of its `tx` (counting the
57//! earlier binds of the file), its blob must be readable, `tx <= asof <= issued`, and a version
58//! at or after its `tx` must be this receipt. Receipts are signed in the first pass (signing is
59//! deterministic), so their hashes are known. A file that the first pass refuses writes nothing.
60
61use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
62use std::io::{BufRead, Write};
63use std::path::{Path, PathBuf};
64use std::sync::atomic::{AtomicBool, Ordering};
65use std::sync::Arc;
66
67use datumesh_core::json::{self, Value};
68use datumesh_core::{
69    sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey,
70    Source, Type,
71};
72use datumesh_engine::{Clock, SystemClock};
73use datumesh_formula::definition::{Callee, Callees, ClassCallee};
74use datumesh_store::{Binding, CellWrite, Namespace, Store};
75
76use crate::testrun::{self, run_test, Overlay, Ran, RowWrite, TestSpec};
77
78use crate::config::ConfigError;
79
80/// The end of time: a binding as of it is the latest one.
81const END: &str = "9999-12-31T23:59:59.999999Z";
82
83/// Cell lines planned together, and cell writes per `put_cells` call (CONTRACT-perf H1, H2).
84const CHUNK: usize = 1_000;
85
86/// The environment variable of the publish lanes (CONTRACT-perf H5), read once per run.
87const LANES_VAR: &str = "DATUMESH_PUBLISH_LANES";
88
89/// The lanes of a run (CONTRACT-perf H5): `DATUMESH_PUBLISH_LANES` (a positive integer), else
90/// `min(4, available_parallelism)`, at least 1. An invalid value refuses the run before any write.
91fn publish_lanes() -> Result<usize, PublishError> {
92    let bad = |detail: String| PublishError::at(0, ConfigError::new(LANES_VAR, detail).to_string());
93    match std::env::var(LANES_VAR) {
94        Err(std::env::VarError::NotPresent) => Ok(std::thread::available_parallelism().map_or(1, |n| n.get()).min(4)),
95        Ok(v) => match v.parse::<usize>() {
96            Ok(n) if n > 0 => Ok(n),
97            _ => Err(bad(format!("must be a positive integer, not {v:?}"))),
98        },
99        Err(e) => Err(bad(e.to_string())),
100    }
101}
102
103/// What a run put.
104#[non_exhaustive]
105#[derive(Clone, Debug, Default, PartialEq, Eq)]
106pub struct Published {
107    /// Functions newly bound (stored and formula).
108    pub functions: usize,
109    /// Classes newly bound (since 0.2.2).
110    pub classes: usize,
111    /// Cell lines read.
112    pub cells: usize,
113    /// Receipts stored (new or already there).
114    pub receipts: usize,
115    /// Versions newly added.
116    pub versions: usize,
117    /// Blobs put (new or already there).
118    pub blobs: usize,
119    /// Blob bytes put.
120    pub blob_bytes: u64,
121    /// Descriptors newly put by descriptor lines (since 0.3.1); a re-run counts 0.
122    pub descriptors: u64,
123}
124
125/// How a run reads the clock and whether it backfills (CONTRACT-1.8b H1).
126#[derive(Clone)]
127pub struct PublishOptions {
128    /// The mesh clock: the start of a binding left out is its now, and a start earlier than it is
129    /// refused (rule A).
130    pub clock: Arc<dyn Clock + Send + Sync>,
131    /// The operator log of a `--backfill` run (`DATUMESH_BACKFILL_LOG`). `Some` lifts the start
132    /// check, for a file whose names and namespaces have no bindings yet, and appends a line to
133    /// the log for each run that writes.
134    pub backfill: Option<PathBuf>,
135}
136
137impl PublishOptions {
138    /// The system clock, and no backfill.
139    pub fn system() -> PublishOptions {
140        PublishOptions { clock: Arc::new(SystemClock), backfill: None }
141    }
142}
143
144/// Why a line (or the file, at line 0) was refused.
145#[non_exhaustive]
146#[derive(Clone, Debug, PartialEq, Eq)]
147pub enum PublishErrorKind {
148    /// Any other refusal: [`PublishError::detail`] says what.
149    Line,
150    /// A binding's start is earlier than the mesh clock (rule A).
151    StartBeforeNow {
152        /// The name (or class name, or method name) that would bind.
153        name: String,
154        /// Its start.
155        start: String,
156        /// The mesh clock.
157        now: String,
158    },
159    /// A `--backfill` run that would write over a binding, or under a namespace with one.
160    Backfill(String),
161    /// `import-vectors`: a name the vectors would bind is under a namespace the mesh owns.
162    OwnedVectorName(String),
163    /// The public rule (D8, Build 2 Part 3, `publish_stored_checked`): a computed function's tests
164    /// do not cover the derived cases or every branch. The text lists what is missing.
165    TestsMissing(String),
166    /// The public rule: a test of a computed function fails. The text names the case and the
167    /// numbers.
168    TestsFailing(String),
169}
170
171impl std::fmt::Display for PublishErrorKind {
172    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
173        match self {
174            PublishErrorKind::Line => f.write_str("refused"),
175            PublishErrorKind::StartBeforeNow { name, start, now } => {
176                write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
177            }
178            PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
179            PublishErrorKind::OwnedVectorName(name) => {
180                write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
181            }
182            PublishErrorKind::TestsMissing(text) => write!(f, "E_TESTS_MISSING: {text}"),
183            PublishErrorKind::TestsFailing(text) => write!(f, "E_TESTS_FAILING: {text}"),
184        }
185    }
186}
187
188/// Why a run stopped: the line, what was wrong, and its typed kind.
189#[derive(Clone, Debug, PartialEq, Eq)]
190pub struct PublishError {
191    /// The line number (1-based; 0 for the file).
192    pub line: usize,
193    /// Why, in words.
194    pub detail: String,
195    /// Why, typed.
196    pub kind: PublishErrorKind,
197}
198
199impl PublishError {
200    fn at(line: usize, detail: impl Into<String>) -> PublishError {
201        PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
202    }
203
204    pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
205        PublishError { line, detail: kind.to_string(), kind }
206    }
207}
208
209impl std::fmt::Display for PublishError {
210    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
211        write!(f, "line {}: {}", self.line, self.detail)
212    }
213}
214
215impl std::error::Error for PublishError {}
216
217fn s(v: &str) -> Value {
218    Value::String(v.to_string())
219}
220
221/// The namespace of a name: the name minus its last `/` segment.
222fn namespace_of(name: &str) -> &str {
223    name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
224}
225
226/// The host of a name: its first segment.
227fn host_of(name: &str) -> &str {
228    name.split('/').next().unwrap_or("")
229}
230
231/// The definer of a class `name`: its own namespace when the mesh owns the name's host, else the
232/// mesh's host (which no class name under another host has, so core answers `E_CLASS_OWNER`).
233fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
234    let host_of_name = name.split('/').next().unwrap_or("");
235    if owned.iter().any(|o| o == host_of_name) {
236        namespace_of(name)
237    } else {
238        host
239    }
240}
241
242/// The start of a line's binding: its `bound`, or the clock's now when it is left out.
243fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
244    match v.get("bound") {
245        None => Ok(now.to_string()),
246        Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
247    }
248}
249
250/// The bindings and versions a pass one has planned so far: the file's own, which are seen before
251/// the store's.
252#[derive(Default)]
253struct Simulation {
254    /// Function name to the (tx, function hash) of each bind the file would make.
255    functions: BTreeMap<String, Vec<(String, String)>>,
256    /// Class name to the (tx, class hash) of each class bind the file would make.
257    classes: BTreeMap<String, Vec<(String, String)>>,
258    /// Canonical bytes by function hash, of the descriptors the file would store.
259    descriptors: BTreeMap<String, Vec<u8>>,
260    /// Canonical bytes by class hash, of the classes the file would store.
261    class_bytes: BTreeMap<String, Vec<u8>>,
262    /// Cell id to the (tx, receipt hash) of each version the file would add, in tx order.
263    versions: BTreeMap<String, Vec<(String, String)>>,
264}
265
266impl Simulation {
267    /// The latest of `binds`' entries for `name` with tx <= `at`.
268    fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
269        binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
270    }
271}
272
273/// The callees of a definition: the store's bindings, and with `sim` the file's own over them.
274struct Callers<'a, S: Store + ?Sized> {
275    store: &'a S,
276    sim: Option<&'a Simulation>,
277}
278
279impl<S: Store + ?Sized> Callers<'_, S> {
280    fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
281        if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
282            return Ok(Some(b.clone()));
283        }
284        Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
285    }
286
287    fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
288        if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
289            return Ok(Some(b.clone()));
290        }
291        self.store.class(hash).map_err(|e| e.to_string())
292    }
293}
294
295impl<S: Store + ?Sized> Callees for Callers<'_, S> {
296    fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
297        let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
298        let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
299        let (fn_hash, start) = match (stored, simmed) {
300            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
301            (Some(b), _) => (b.fn_hash, b.tx),
302            (None, Some((t, h))) => (h, t),
303            (None, None) => return Ok(None),
304        };
305        let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
306        let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
307        Ok(Some(Callee { fn_hash, start, descriptor }))
308    }
309
310    fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
311        let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
312        let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
313        let (class_hash, start) = match (stored, simmed) {
314            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
315            (Some(b), _) => (b.class_hash, b.tx),
316            (None, Some((t, h))) => (h, t),
317            (None, None) => return Ok(None),
318        };
319        let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
320        let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
321        Ok(Some(ClassCallee { class_hash, start, descriptor }))
322    }
323
324    fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
325        let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
326        ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
327    }
328
329    fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
330        let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
331        json::parse(&bytes).map(Some).map_err(|e| e.to_string())
332    }
333}
334
335/// What a run reads from the store again and again, kept for the run (CONTRACT-perf H1). The
336/// store does not change during the first pass, so each answer is the same whenever it is asked,
337/// and a failed read is kept as its message, which is what the check that asks it reports. A
338/// prefetch that fails keeps nothing: each line then asks for itself and fails at its own line.
339#[derive(Default)]
340struct Reads {
341    bindings: HashMap<(String, String), Result<Option<Binding>, String>>,
342    descriptors: HashMap<String, Option<Vec<u8>>>,
343    namespaces: HashMap<String, Result<Option<Namespace>, String>>,
344    /// The store's latest version of a cell (tx, receipt hash), prefetched per chunk.
345    latest_of: HashMap<String, Option<(String, String)>>,
346    /// The store's receipt, prefetched for the latest receipts a skip check reads.
347    receipt_of: HashMap<String, Option<Vec<u8>>>,
348}
349
350impl Reads {
351    fn binding<S: Store + ?Sized>(&mut self, store: &S, name: &str, at: &str) -> Result<Option<Binding>, String> {
352        let key = (name.to_string(), at.to_string());
353        if let Some(r) = self.bindings.get(&key) {
354            return r.clone();
355        }
356        let r = store.resolve_binding(name, at).map_err(|e| e.to_string());
357        self.bindings.insert(key, r.clone());
358        r
359    }
360
361    fn descriptor<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
362        if let Some(b) = self.descriptors.get(hash) {
363            return b.clone();
364        }
365        let b = store.descriptor(hash).map(|b| b.into_owned());
366        self.descriptors.insert(hash.to_string(), b.clone());
367        b
368    }
369
370    fn namespace<S: Store + ?Sized>(&mut self, store: &S, host: &str) -> Result<Option<Namespace>, String> {
371        if let Some(r) = self.namespaces.get(host) {
372            return r.clone();
373        }
374        let r = store.namespace(host).map_err(|e| e.to_string());
375        self.namespaces.insert(host.to_string(), r.clone());
376        r
377    }
378
379    /// The store's latest version of `id`: prefetched, else asked now.
380    fn latest<S: Store + ?Sized>(&mut self, store: &S, id: &str) -> Result<Option<(String, String)>, String> {
381        if let Some(l) = self.latest_of.get(id) {
382            return Ok(l.clone());
383        }
384        store.latest(id).map(|v| v.map(|v| (v.tx, v.receipt_hash))).map_err(|e| e.to_string())
385    }
386
387    /// The stored receipt `hash`, as `receipt` answers: prefetched, else asked now.
388    fn receipt<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
389        if let Some(b) = self.receipt_of.get(hash) {
390            return b.clone();
391        }
392        store.receipt(hash).map(|b| b.into_owned())
393    }
394
395    /// One `latest_many` for the ids not read yet.
396    fn prefetch_latest<S: Store + ?Sized>(&mut self, store: &S, ids: &[&str]) {
397        let mut seen = HashSet::new();
398        let missing: Vec<&str> =
399            ids.iter().copied().filter(|id| !self.latest_of.contains_key(*id) && seen.insert(*id)).collect();
400        if missing.is_empty() {
401            return;
402        }
403        if let Ok(found) = store.latest_many(&missing) {
404            for (id, v) in missing.iter().zip(found) {
405                self.latest_of.insert(id.to_string(), v.map(|v| (v.tx, v.receipt_hash)));
406            }
407        }
408    }
409
410    /// One `receipts_many` for the hashes not read yet.
411    fn prefetch_receipts<S: Store + ?Sized>(&mut self, store: &S, hashes: &[&str]) {
412        let mut seen = HashSet::new();
413        let missing: Vec<&str> =
414            hashes.iter().copied().filter(|h| !self.receipt_of.contains_key(*h) && seen.insert(*h)).collect();
415        if missing.is_empty() {
416            return;
417        }
418        if let Ok(found) = store.receipts_many(&missing) {
419            for (h, b) in missing.iter().zip(found) {
420                self.receipt_of.insert(h.to_string(), b);
421            }
422        }
423    }
424}
425
426/// `f` over `items` on up to `available_parallelism()` threads; the results in item order.
427fn parallel_owned<T: Send, R: Send>(items: Vec<T>, f: impl Fn(T) -> R + Sync) -> Vec<R> {
428    let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(items.len());
429    if threads <= 1 {
430        return items.into_iter().map(f).collect();
431    }
432    let size = items.len().div_ceil(threads);
433    let mut parts: Vec<Vec<T>> = Vec::with_capacity(threads);
434    let mut rest = items.into_iter();
435    loop {
436        let part: Vec<T> = rest.by_ref().take(size).collect();
437        if part.is_empty() {
438            break;
439        }
440        parts.push(part);
441    }
442    std::thread::scope(|s| {
443        let handles: Vec<_> = parts
444            .into_iter()
445            .map(|part| {
446                let f = &f;
447                s.spawn(move || part.into_iter().map(f).collect::<Vec<R>>())
448            })
449            .collect();
450        handles.into_iter().flat_map(|h| h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect()
451    })
452}
453
454/// The function hash `name` means at the end of time: the file's own over the store's.
455fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
456    if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
457        return Ok(Some(h));
458    }
459    store.resolve_name(name, END).map_err(|e| e.to_string())
460}
461
462/// The class hash `name` means at the end of time: the file's own over the store's.
463fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
464    if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
465        return Ok(Some(h));
466    }
467    store.resolve_class(name, END).map_err(|e| e.to_string())
468}
469
470/// The function hash `name` means as of `at`, counting the file's earlier binds.
471fn function_at<S: Store + ?Sized>(
472    reads: &mut Reads,
473    store: &S,
474    sim: &Simulation,
475    name: &str,
476    at: &str,
477) -> Result<Option<String>, String> {
478    let stored = reads.binding(store, name, at)?;
479    let simmed = Simulation::latest(&sim.functions, name, at);
480    Ok(match (stored, simmed) {
481        (Some(b), Some((t, h))) if t >= b.tx => Some(h),
482        (Some(b), _) => Some(b.fn_hash),
483        (None, Some((_, h))) => Some(h),
484        (None, None) => None,
485    })
486}
487
488/// The latest binding tx of `name` (a class's, or a function's), the store's or the file's.
489fn latest_bind<S: Store + ?Sized>(
490    store: &S,
491    sim: &Simulation,
492    name: &str,
493    class: bool,
494) -> Result<Option<String>, String> {
495    let stored: Option<String> = if class {
496        store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
497    } else {
498        store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
499    };
500    let binds = if class { &sim.classes } else { &sim.functions };
501    let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
502    Ok(match (stored, planned) {
503        (Some(a), Some(b)) => Some(a.max(b)),
504        (a, b) => a.or(b),
505    })
506}
507
508/// Whether `name` is bound as the other kind: a function for a class, a class for a function.
509fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
510    if class {
511        Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
512    } else {
513        Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
514            || sim.classes.contains_key(name))
515    }
516}
517
518/// The latest version of a cell, the store's or the file's: its tx and receipt hash.
519fn cell_latest<S: Store + ?Sized>(
520    reads: &mut Reads,
521    store: &S,
522    sim: &Simulation,
523    id: &str,
524) -> Result<Option<(String, String)>, String> {
525    let stored = reads.latest(store, id)?;
526    let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
527    Ok(match (stored, planned) {
528        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
529        (a, b) => a.or(b),
530    })
531}
532
533/// The receipt hash of a cell's version as of `at`, the store's or the file's.
534fn cell_version_asof<S: Store + ?Sized>(
535    store: &S,
536    sim: &Simulation,
537    id: &str,
538    at: &str,
539) -> Result<Option<String>, String> {
540    let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
541    let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
542    let latest = match (stored, planned) {
543        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
544        (a, b) => a.or(b),
545    };
546    Ok(latest.map(|x| x.1))
547}
548
549/// The descriptor bytes of a function hash: the file's own, else the store's (memoized).
550fn function_bytes_of<S: Store + ?Sized>(
551    reads: &mut Reads,
552    store: &S,
553    sim: &Simulation,
554    hash: &str,
555) -> Result<Option<Vec<u8>>, String> {
556    if let Some(b) = sim.descriptors.get(hash) {
557        return Ok(Some(b.clone()));
558    }
559    Ok(reads.descriptor(store, hash))
560}
561
562/// Whether a namespace is suspended (the store refuses binds and class puts under it).
563fn suspended<S: Store + ?Sized>(reads: &mut Reads, store: &S, host: &str, line: usize) -> Result<(), PublishError> {
564    match reads.namespace(store, host) {
565        Ok(Some(n)) if n.suspended_at.is_some() => {
566            Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
567        }
568        Ok(_) => Ok(()),
569        Err(e) => Err(PublishError::at(line, e)),
570    }
571}
572
573/// A binding a run would make: the line that makes it and its name.
574struct Bind {
575    line: usize,
576    name: String,
577}
578
579/// What a run writes, for the backfill log: the namespaces and starts of its binds.
580#[derive(Default)]
581struct Written {
582    namespaces: BTreeSet<String>,
583    starts: BTreeSet<String>,
584}
585
586impl Written {
587    fn bound(&mut self, name: &str, start: &str) {
588        self.namespaces.insert(namespace_of(name).to_string());
589        self.starts.insert(start.to_string());
590    }
591}
592
593/// The per-run values both passes use.
594struct Ctx<'a> {
595    host: &'a str,
596    owned: &'a [String],
597    /// The start of the mesh's key.
598    key_from: &'a str,
599    /// The clock's now, read once for the run.
600    now: &'a str,
601    /// The directory of the file (blobs are relative to it).
602    base: &'a Path,
603}
604
605/// One write the second pass makes, planned (and checked) by the first.
606enum Op {
607    /// A descriptor line: put; counted when newly put.
608    Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
609    /// A function (stored or formula): put its descriptor, and bind the name when `bind`.
610    Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
611    /// A class: put it, and bind it (its methods with it, in the store) when `bind`.
612    Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
613    /// A cell: its blob, its receipt, and the version when `version`.
614    Cell { tx: String, id: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
615    /// A catalogue line (R5): the entry, and its dictionaries (`publisher`, signed by the run's key).
616    Catalog { entry: datumesh_store::CatalogEntry, dictionaries: Vec<(String, Vec<datumesh_store::KeyValue>)> },
617}
618
619/// The lines of a file, numbered, without the blank ones.
620fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
621    let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
622    let mut out = Vec::new();
623    for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
624        let n = i + 1;
625        let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
626        if !line.trim().is_empty() {
627            out.push((n, line));
628        }
629    }
630    Ok(out)
631}
632
633/// The dictionary values the file's cells have for the string arguments (CONTRACT-build2-part2-read
634/// R3): each distinct (function, key, value) that no dictionary has yet is stored with an empty label
635/// (the value stands for itself). A dictionary's label is never replaced by a cell's.
636fn index_cell_values<S: Store + ?Sized>(store: &S, ops: &[Op]) -> Result<(), String> {
637    let mut seen = BTreeSet::new();
638    for op in ops {
639        if let Op::Cell { receipt, .. } = op {
640            if let Ok(r) = datumesh_core::Receipt::from_json(receipt) {
641                cell_string_values(&r, &mut seen);
642            }
643        }
644    }
645    store_cell_values(store, seen)
646}
647
648/// The (function, key, value) of each string argument of a receipt's cell, into `out`.
649pub(crate) fn cell_string_values(r: &datumesh_core::Receipt, out: &mut BTreeSet<(String, String, String)>) {
650    let cell = r.cell();
651    for (key, arg) in &cell.args {
652        if let datumesh_core::Arg::Value(datumesh_core::Scalar::String(v)) = arg {
653            if !v.is_empty() && v.len() <= 1024 {
654                out.insert((cell.function_name(), key.clone(), v.clone()));
655            }
656        }
657    }
658}
659
660/// Stores the values of [`cell_string_values`] that no dictionary has yet (R3).
661pub(crate) fn store_cell_values<S: Store + ?Sized>(
662    store: &S,
663    seen: BTreeSet<(String, String, String)>,
664) -> Result<(), String> {
665    for (owner, key, value) in seen {
666        if crate::find::has_value(store, &owner, &key, &value) {
667            continue;
668        }
669        let kv = datumesh_store::KeyValue { value, label: String::new(), aliases: Vec::new() };
670        store.upsert_key_values(&owner, &key, std::slice::from_ref(&kv)).map_err(|e| e.to_string())?;
671    }
672    Ok(())
673}
674
675/// Whether a line is a cell line: none of the other kinds (the dispatch order of `plan_line`).
676fn is_cell_line(v: &Value) -> bool {
677    v.get("class").and_then(Value::as_str).is_none()
678        && v.get("descriptor").is_none()
679        && v.get("function").and_then(Value::as_str).is_none()
680        && v.get("formula").and_then(Value::as_str).is_none()
681        && v.get("cell").is_some()
682}
683
684/// A cell line prepared for signing: everything its receipt takes, checked up to there.
685struct Pending {
686    line: usize,
687    ns: String,
688    cell: Cell,
689    value: Scalar,
690    valid: Args,
691    tx: String,
692    asof: String,
693    issued: String,
694    has_tx: bool,
695    licence: String,
696    summary: String,
697    blob: Option<Vec<u8>>,
698    /// The `source` of a relayed value (spec-0.6.0, 8.1), as the line gives it.
699    source: Option<Source>,
700}
701
702/// The `source` object of a cell line (spec-0.6.0, C2): exactly `name`, `url` and `retrieved`, with
703/// `sha256` and `takeover` optional, all strings. Its rules (a DNS name, an instant, an `https://`
704/// takeover, a publisher that does not relay itself) are the receipt's, checked when it is signed.
705fn source_of_line(v: &Value) -> Result<Source, String> {
706    let Value::Object(m) = v else { return Err("source is an object".into()) };
707    for (k, x) in m {
708        if !matches!(k.as_str(), "name" | "url" | "retrieved" | "sha256" | "takeover") {
709            return Err(format!("source has no member {k}"));
710        }
711        if !matches!(x, Value::String(_)) {
712            return Err(format!("source.{k} is a string"));
713        }
714    }
715    let text = |k: &str| -> Result<&str, String> {
716        v.get(k).and_then(Value::as_str).ok_or_else(|| format!("source has no {k}"))
717    };
718    let mut out = Source::new(text("name")?, text("url")?, text("retrieved")?);
719    if let Some(h) = v.get("sha256").and_then(Value::as_str) {
720        out = out.with_sha256(h);
721    }
722    if let Some(t) = v.get("takeover").and_then(Value::as_str) {
723        out = out.with_takeover(t);
724    }
725    Ok(out)
726}
727
728impl Pending {
729    /// Signs the receipt (deterministic) and takes its hash and cell id. Runs on any thread.
730    fn sign(self, key: &SigningKey, host: &str) -> Result<Signed, PublishError> {
731        let line = self.line;
732        let err = move |d: String| PublishError::at(line, d);
733        let b = ReceiptBuilder::new()
734            .cell(self.cell.clone())
735            .value(self.value)
736            .valid(self.valid)
737            .asof(self.asof.as_str())
738            .tx(self.tx.as_str())
739            .issued(self.issued.as_str())
740            .publisher(self.ns.as_str())
741            .signer_host(host)
742            .licence(self.licence.as_str())
743            .summary(self.summary.as_str());
744        let b = match self.source {
745            Some(s) => b.source(s),
746            None => b,
747        };
748        let r = b.sign(key).map_err(|e| err(e.to_string()))?;
749        let receipt = receipt_json(&r);
750        let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
751        let id = self.cell.id();
752        Ok(Signed { line, has_tx: self.has_tx, tx: self.tx, id, hash, receipt, blob: self.blob })
753    }
754}
755
756/// A cell line signed: what its checks and its write take.
757struct Signed {
758    line: usize,
759    has_tx: bool,
760    tx: String,
761    id: String,
762    hash: String,
763    receipt: Vec<u8>,
764    blob: Option<Vec<u8>>,
765}
766
767/// Pass one: plans the writes of the lines and checks each as the store will (CONTRACT-1.7a H2).
768struct Planner<'a, S: Store + ?Sized> {
769    store: &'a S,
770    key: &'a SigningKey,
771    ctx: Ctx<'a>,
772    backfill: bool,
773    sim: Simulation,
774    reads: Reads,
775    ops: Vec<Op>,
776    binds: Vec<Bind>,
777    /// The test lines of the file, with their lines (Build 2 Part 3).
778    tests: Vec<(usize, TestSpec)>,
779    /// The computed functions the file defines: (line, name, function hash).
780    formulas: Vec<(usize, String, String)>,
781}
782
783impl<S: Store + ?Sized> Planner<'_, S> {
784    fn callers(&self) -> Callers<'_, S> {
785        Callers { store: self.store, sim: Some(&self.sim) }
786    }
787
788    /// Plans the lines in file order. Cell lines are planned in chunks of up to `CHUNK`
789    /// consecutive ones; any other line ends a chunk (CONTRACT-perf H1).
790    fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
791        let mut cells: Vec<(usize, Value)> = Vec::new();
792        for (n, text) in lines {
793            let n = *n;
794            match json::parse(text.as_bytes()).map_err(|e| PublishError::at(n, e.to_string())) {
795                Ok(v) if is_cell_line(&v) => {
796                    cells.push((n, v));
797                    if cells.len() == CHUNK {
798                        self.plan_cells(std::mem::take(&mut cells))?;
799                    }
800                }
801                other => {
802                    // The cells before this line are checked first: a failure of either is the
803                    // file's first.
804                    self.plan_cells(std::mem::take(&mut cells))?;
805                    self.plan_line(n, &other?)?;
806                }
807            }
808        }
809        self.plan_cells(cells)
810    }
811
812    /// One line that is not a cell (the dispatch of the lines before cells, as it was).
813    fn plan_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
814        let err = |d: String| PublishError::at(n, d);
815        if let Some(name) = v.get("class").and_then(Value::as_str) {
816            self.class_line(n, name, v)?;
817        } else if let Some(d) = v.get("descriptor") {
818            self.descriptor_line(n, d)?;
819        } else if let Some(name) = v.get("function").and_then(Value::as_str) {
820            let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
821            let start = bound_of(v, n, self.ctx.now)?;
822            let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
823            self.function(n, name, start, fd.hash(), fd.canonical())?;
824        } else if v.get("catalog").is_some() {
825            self.catalog_line(n, v)?;
826        } else if let Some(name) = v.get("formula").and_then(Value::as_str) {
827            let source =
828                v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
829            let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
830            let start = bound_of(v, n, self.ctx.now)?;
831            let def = datumesh_formula::definition::define_function(
832                name,
833                source,
834                sig,
835                &Default::default(),
836                &self.callers(),
837                &start,
838            )
839            .map_err(|e| err(format!("define {name}: {e}")))?;
840            let hash = def.fn_hash.clone();
841            self.function(n, name, start, def.fn_hash, def.bytes)?;
842            self.formulas.push((n, name.to_string(), hash));
843        } else if v.get("test").is_some() {
844            let spec = testrun::parse_test(v).map_err(err)?;
845            self.tests.push((n, spec));
846        } else {
847            return Err(err("neither a function nor a cell".into()));
848        }
849        Ok(())
850    }
851
852    /// `{"catalog": <name>, "kind", "label", "description", "args", "examples", "dictionary"}` (R5):
853    /// the publisher's text for a function or class bound in this file or the store, under a host
854    /// this mesh owns. Checked here, so a refused line writes nothing.
855    fn catalog_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
856        let err = |d: String| PublishError::at(n, d);
857        let d = crate::catalog::parse(v, "catalog").map_err(err)?;
858        let host = host_of(&d.name);
859        if host != self.ctx.host && !self.ctx.owned.iter().any(|o| o == host) {
860            return Err(err(format!("{} is not under a host this mesh owns", d.name)));
861        }
862        let hash = match d.kind {
863            datumesh_store::CatalogKind::Class => current_class(self.store, &self.sim, &d.name).map_err(err)?,
864            _ => current_function(self.store, &self.sim, &d.name).map_err(err)?,
865        };
866        if hash.is_none() {
867            return Err(err(format!("{} is bound nowhere", d.name)));
868        }
869        let entry = d.entry(hash, datumesh_store::CatalogSource::Publisher, Some(self.key.key_id().to_string()));
870        self.ops.push(Op::Catalog { entry, dictionaries: d.dictionary });
871        Ok(())
872    }
873
874    /// `{"descriptor": ...}`: a stored function's descriptor, put and bound to nothing.
875    fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
876        let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
877        if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
878            return Err(PublishError::at(
879                n,
880                "a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
881            ));
882        }
883        let (hash, bytes) = (fd.hash(), fd.canonical());
884        self.sim.descriptors.insert(hash.clone(), bytes.clone());
885        self.ops.push(Op::Descriptor { hash, bytes, counted: true });
886        Ok(())
887    }
888
889    /// A function's descriptor (stored or formula): put, and bound when its name does not mean it.
890    fn function(
891        &mut self,
892        n: usize,
893        name: &str,
894        start: String,
895        hash: String,
896        bytes: Vec<u8>,
897    ) -> Result<(), PublishError> {
898        self.sim.descriptors.insert(hash.clone(), bytes.clone());
899        let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
900        let bind = current.as_deref() != Some(hash.as_str());
901        if bind {
902            self.check_bind(n, name, &start, false)?;
903            self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
904        }
905        self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
906        Ok(())
907    }
908
909    fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
910        let err = |d: String| PublishError::at(n, d);
911        let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
912        let start = bound_of(v, n, self.ctx.now)?;
913        let def = datumesh_formula::definition::define_class(
914            name,
915            &descriptor.to_jcs(),
916            definer(name, self.ctx.host, self.ctx.owned),
917            &self.callers(),
918            &start,
919        )
920        .map_err(|e| err(format!("define {name}: {e}")))?;
921        // put_class refuses under a suspended steward namespace, whether or not the name binds. A
922        // namespace row is a host's (9.9), so the steward namespace's host is asked: a steward with
923        // a path (datumesh.com/scripture) is not a host and Store::namespace refuses it.
924        suspended(&mut self.reads, self.store, host_of(&def.descriptor.steward.namespace), n)?;
925        let hash = def.class_hash.clone();
926        let methods: Vec<(String, String)> =
927            def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
928        let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
929        if bind {
930            self.check_bind(n, name, &start, true)?;
931            // bind_class binds every method in the same transaction (the store's rule).
932            for (mname, _) in &methods {
933                self.check_bind(n, mname, &start, false)?;
934            }
935            self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
936            for (mname, fh) in &methods {
937                self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
938            }
939        }
940        self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
941        self.ops.push(Op::Class {
942            name: name.to_string(),
943            start,
944            hash,
945            bytes: def.bytes,
946            bind,
947            methods: methods.into_iter().map(|(m, _)| m).collect(),
948        });
949        Ok(())
950    }
951
952    /// One bind a line would make: rule A, the other kind, the namespace, and the latest binding.
953    fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
954        if !self.backfill && start < self.ctx.now {
955            return Err(PublishError::typed(
956                line,
957                PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
958            ));
959        }
960        let err = |d: String| PublishError::at(line, d);
961        if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
962            let kind = if class { "function" } else { "class" };
963            return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
964        }
965        suspended(&mut self.reads, self.store, host_of(name), line)?;
966        if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
967            if start <= latest.as_str() {
968                return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
969            }
970        }
971        self.binds.push(Bind { line, name: name.into() });
972        Ok(())
973    }
974
975    /// Plans the cell lines of a chunk (CONTRACT-perf H1): each is prepared in file order up to its
976    /// receipt; the receipts are signed on threads; the store is read once for the chunk; then each
977    /// is checked in file order. The first failure is the chunk's, and a line that fails before
978    /// signing is reported after the lines before it are checked.
979    fn plan_cells(&mut self, lines: Vec<(usize, Value)>) -> Result<(), PublishError> {
980        if lines.is_empty() {
981            return Ok(());
982        }
983        let mut pending = Vec::with_capacity(lines.len());
984        let mut failed = None;
985        for (n, v) in &lines {
986            match self.prepare_cell(*n, v) {
987                Ok(p) => pending.push(p),
988                Err(e) => {
989                    failed = Some(e);
990                    break;
991                }
992            }
993        }
994        let (key, host) = (self.key, self.ctx.host);
995        let signed = parallel_owned(pending, |p| p.sign(key, host));
996        let ids: Vec<&str> = signed.iter().flatten().map(|s| s.id.as_str()).collect();
997        self.reads.prefetch_latest(self.store, &ids);
998        let hashes: Vec<String> = signed
999            .iter()
1000            .flatten()
1001            .filter(|s| !s.has_tx)
1002            .filter_map(|s| self.reads.latest_of.get(&s.id).cloned().flatten())
1003            .map(|(_, h)| h)
1004            .collect();
1005        let hashes: Vec<&str> = hashes.iter().map(String::as_str).collect();
1006        self.reads.prefetch_receipts(self.store, &hashes);
1007        for s in signed {
1008            self.check_cell(s?)?;
1009        }
1010        match failed {
1011            Some(e) => Err(e),
1012            None => Ok(()),
1013        }
1014    }
1015
1016    /// A cell line up to its receipt, as the checks of a line take them in order: the fields, the
1017    /// instants, the function as of `tx`, the descriptor, the arguments and the value (and its blob).
1018    fn prepare_cell(&mut self, n: usize, v: &Value) -> Result<Pending, PublishError> {
1019        let err = |d: String| PublishError::at(n, d);
1020        let cell_v = v.get("cell").ok_or_else(|| err("a cell without fn".into()))?;
1021        let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
1022        let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
1023        let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
1024        let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
1025        let (licence, summary) = (required("licence")?, required("summary")?);
1026        // tx and asof default to the run's clock now, each on its own (H4).
1027        let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
1028        let has_tx = field("tx").is_some();
1029        let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
1030        if tx > asof {
1031            return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
1032        }
1033        let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
1034        if asof > issued {
1035            return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
1036        }
1037        let fn_hash = function_at(&mut self.reads, self.store, &self.sim, name, &tx)
1038            .map_err(err)?
1039            .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
1040        let d = function_bytes_of(&mut self.reads, self.store, &self.sim, &fn_hash)
1041            .map_err(err)?
1042            .ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
1043        let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
1044        let mut args = Args::new();
1045        for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
1046            args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
1047        }
1048        let valid: Args = args
1049            .iter()
1050            .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
1051            .map(|(k, a)| (k.clone(), a.clone()))
1052            .collect();
1053        let source = match v.get("source") {
1054            Some(sv) => Some(source_of_line(sv).map_err(err)?),
1055            None => None,
1056        };
1057        let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
1058        let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
1059            Some(file) => {
1060                let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
1061                let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
1062                let sha = sha256_hex(&bytes);
1063                let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
1064                    .map_err(|e| err(e.to_string()))?;
1065                (scalar, Some(bytes))
1066            }
1067            None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
1068        };
1069        let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
1070        Ok(Pending {
1071            line: n,
1072            ns: ns.to_string(),
1073            cell,
1074            value,
1075            valid,
1076            tx,
1077            asof,
1078            issued,
1079            has_tx,
1080            licence,
1081            summary,
1082            blob,
1083            source,
1084        })
1085    }
1086
1087    /// The checks of a signed cell line, in file order: the skip rule of a line without `tx`, then
1088    /// the version rule. Its write is planned.
1089    fn check_cell(&mut self, s: Signed) -> Result<(), PublishError> {
1090        let line = s.line;
1091        let err = |d: String| PublishError::at(line, d);
1092        // A line without tx takes the run's clock, so a re-run at a later clock would sign the same
1093        // fact again as a new version. Re-running a file changes nothing: when the cell's latest
1094        // version is the store's and states the same thing (type, value, valid, licence, summary),
1095        // the line is skipped. A changed value is a new version, a correction.
1096        if !s.has_tx {
1097            if let Some((_, latest)) = cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1098                if let Some(old) = self.reads.receipt(self.store, &latest) {
1099                    let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
1100                    let new = json::parse(&s.receipt).map_err(|e| err(e.to_string()))?;
1101                    if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
1102                        return Ok(());
1103                    }
1104                }
1105            }
1106        }
1107        let version = match cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1108            Some((latest_tx, _)) if latest_tx.as_str() >= s.tx.as_str() => {
1109                if cell_version_asof(self.store, &self.sim, &s.id, &s.tx).map_err(err)?.as_deref()
1110                    != Some(s.hash.as_str())
1111                {
1112                    return Err(err(format!("{} has a version at or after {} that is not this receipt", s.id, s.tx)));
1113                }
1114                false
1115            }
1116            _ => {
1117                self.sim.versions.entry(s.id.clone()).or_default().push((s.tx.clone(), s.hash.clone()));
1118                true
1119            }
1120        };
1121        self.ops.push(Op::Cell { tx: s.tx, id: s.id, receipt: s.receipt, blob: s.blob, version });
1122        Ok(())
1123    }
1124}
1125
1126/// The stored descriptor of a function line's signature.
1127fn stored_value(signature: &Value) -> Value {
1128    Value::Object(vec![
1129        ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
1130        ("inputs".into(), Value::Array(vec![])),
1131        ("signature".into(), signature.clone()),
1132    ])
1133}
1134
1135/// The backfill's checks (since 0.3.0 with store 0.4.2): no name the file binds has a binding, and
1136/// no namespace of one has any binding (`has_bindings_under`).
1137fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
1138    for b in binds {
1139        let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
1140        if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
1141            return Err(refuse(format!("{} has a binding", b.name)));
1142        }
1143        let ns = namespace_of(&b.name);
1144        if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
1145            return Err(refuse(format!("{ns} has bindings")));
1146        }
1147    }
1148    Ok(())
1149}
1150
1151/// Appends a run's line to the backfill log (created with mode 0600).
1152fn append_log(log: &Path, written: &Written) -> Result<(), String> {
1153    let line = Value::Object(vec![
1154        ("at".into(), s(&SystemClock.now())),
1155        ("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
1156        ("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
1157    ])
1158    .to_jcs();
1159    let mut opts = std::fs::OpenOptions::new();
1160    opts.create(true).append(true);
1161    #[cfg(unix)]
1162    {
1163        use std::os::unix::fs::OpenOptionsExt;
1164        opts.mode(0o600);
1165    }
1166    let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
1167    let mut bytes = line;
1168    bytes.push(b'\n');
1169    f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
1170}
1171
1172/// Pass two: the writes the first pass planned, in order. Nothing here can refuse a planned
1173/// write, so a refused file has written nothing. Consecutive cell writes go in `put_cells` calls
1174/// of up to `CHUNK` on `lanes` lanes, their blobs put first (CONTRACT-perf H2, H5).
1175fn write<S: Store + ?Sized>(
1176    store: &S,
1177    ops: &[Op],
1178    cells: impl Fn(&S, &[Op], &mut Published) -> Result<(), String>,
1179) -> Result<(Published, Written), PublishError> {
1180    let mut out = Published::default();
1181    let mut written = Written::default();
1182    let err = |d: String| PublishError::at(0, d);
1183    let mut i = 0;
1184    while i < ops.len() {
1185        match &ops[i] {
1186            Op::Descriptor { hash, bytes, counted } => {
1187                let existed = store.descriptor(hash).is_some();
1188                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1189                if *counted && !existed {
1190                    out.descriptors += 1;
1191                }
1192            }
1193            Op::Function { name, start, hash, bytes, bind } => {
1194                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1195                if *bind {
1196                    store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1197                    out.functions += 1;
1198                    written.bound(name, start);
1199                }
1200            }
1201            Op::Class { name, start, hash, bytes, bind, methods } => {
1202                store.put_class(bytes).map_err(|e| err(e.to_string()))?;
1203                if *bind {
1204                    store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1205                    out.classes += 1;
1206                    written.bound(name, start);
1207                    for m in methods {
1208                        written.bound(m, start);
1209                    }
1210                }
1211            }
1212            Op::Catalog { entry, dictionaries } => {
1213                crate::catalog::store_publisher(store, entry, dictionaries).map_err(err)?;
1214            }
1215            Op::Cell { .. } => {
1216                let mut j = i;
1217                while j < ops.len() && j - i < CHUNK && matches!(ops[j], Op::Cell { .. }) {
1218                    j += 1;
1219                }
1220                cells(store, &ops[i..j], &mut out).map_err(err)?;
1221                i = j;
1222                continue;
1223            }
1224        }
1225        i += 1;
1226    }
1227    Ok((out, written))
1228}
1229
1230/// A run of cell writes (up to `CHUNK`) on one lane: their blobs first, then the receipts and
1231/// versions in file order in `put_cells` calls of up to `CHUNK` (CONTRACT-perf H2). No `Sync`
1232/// bound: this is what [`publish_stored_with`] runs.
1233fn write_cells_one_lane<S: Store + ?Sized>(store: &S, group: &[Op], out: &mut Published) -> Result<(), String> {
1234    let items = put_blobs_and_items(store, group, out)?;
1235    let failed = AtomicBool::new(false);
1236    let ks: Vec<usize> = (0..items.len()).collect();
1237    if let Some((_, detail)) = put_lane(store, &items, &ks, &failed) {
1238        return Err(detail);
1239    }
1240    count_cells(group, &items, out);
1241    Ok(())
1242}
1243
1244/// A run of cell writes (up to `CHUNK`): their blobs first, then their receipts and versions on
1245/// `lanes` lanes (CONTRACT-perf H5). A cell's id picks its lane (its first hex digit, mod `lanes`),
1246/// so all writes of one cell share a lane, in file order. A lane's `put_cells` calls take up to
1247/// `CHUNK` of its items. When one fails, no lane starts another call, and the error returned is
1248/// that of the earliest failing item in file order. With one lane, as [`write_cells_one_lane`].
1249fn write_cells<S: Store + Sync + ?Sized>(
1250    store: &S,
1251    group: &[Op],
1252    lanes: usize,
1253    out: &mut Published,
1254) -> Result<(), String> {
1255    if lanes <= 1 {
1256        return write_cells_one_lane(store, group, out);
1257    }
1258    let items = put_blobs_and_items(store, group, out)?;
1259    let mut by_lane: Vec<Vec<usize>> = vec![Vec::new(); lanes];
1260    for (k, op) in group.iter().enumerate() {
1261        if let Op::Cell { id, .. } = op {
1262            by_lane[lane_of(id, lanes)].push(k);
1263        }
1264    }
1265    let failed = AtomicBool::new(false);
1266    let failures: Vec<(usize, String)> = std::thread::scope(|scope| {
1267        let handles: Vec<_> = by_lane
1268            .iter()
1269            .map(|ks| {
1270                let (items, failed) = (&items, &failed);
1271                scope.spawn(move || put_lane(store, items, ks, failed))
1272            })
1273            .collect();
1274        handles.into_iter().filter_map(|h| h.join().unwrap_or_else(|p| std::panic::resume_unwind(p))).collect()
1275    });
1276    if let Some((_, detail)) = failures.into_iter().min_by_key(|(k, _)| *k) {
1277        return Err(detail);
1278    }
1279    count_cells(group, &items, out);
1280    Ok(())
1281}
1282
1283/// The blobs of a run, put in order (counted in `out`), and the run's cell writes.
1284fn put_blobs_and_items<'a, S: Store + ?Sized>(
1285    store: &S,
1286    group: &'a [Op],
1287    out: &mut Published,
1288) -> Result<Vec<CellItem<'a>>, String> {
1289    for op in group {
1290        if let Op::Cell { blob: Some(bytes), .. } = op {
1291            store.put_blob(bytes).map_err(|e| e.to_string())?;
1292            out.blobs += 1;
1293            out.blob_bytes += bytes.len() as u64;
1294        }
1295    }
1296    Ok(group
1297        .iter()
1298        .filter_map(|op| match op {
1299            Op::Cell { tx, id, receipt, version, .. } => {
1300                Some((receipt.as_slice(), if *version { Some((id.as_str(), tx.as_str())) } else { None }))
1301            }
1302            _ => None,
1303        })
1304        .collect())
1305}
1306
1307/// Counts a written run of cells in `out`.
1308fn count_cells(group: &[Op], items: &[CellItem<'_>], out: &mut Published) {
1309    out.cells += group.len();
1310    out.receipts += group.len();
1311    out.versions += items.iter().filter(|(_, v)| v.is_some()).count();
1312}
1313
1314/// The lane of a cell id: its first hex digit, mod `lanes` (CONTRACT-perf H5).
1315fn lane_of(id: &str, lanes: usize) -> usize {
1316    id.bytes().next().and_then(|b| (b as char).to_digit(16)).unwrap_or(0) as usize % lanes
1317}
1318
1319/// A cell write of a run: its receipt's bytes and its version (cell id, tx), if any.
1320type CellItem<'a> = (&'a [u8], Option<(&'a str, &'a str)>);
1321
1322/// One lane's writes: its items (indices into `items`, in file order) in `put_cells` calls of up to
1323/// `CHUNK`. Returns the index of the first item of the call that failed, and its error. No call
1324/// starts once `failed` is set.
1325fn put_lane<S: Store + ?Sized>(
1326    store: &S,
1327    items: &[CellItem<'_>],
1328    ks: &[usize],
1329    failed: &AtomicBool,
1330) -> Option<(usize, String)> {
1331    for call in ks.chunks(CHUNK) {
1332        if failed.load(Ordering::SeqCst) {
1333            return None;
1334        }
1335        let writes: Vec<CellWrite<'_>> = call.iter().map(|&k| CellWrite::new(items[k].0, items[k].1)).collect();
1336        if let Err(e) = store.put_cells(&writes) {
1337            failed.store(true, Ordering::SeqCst);
1338            return Some((call[0], e.to_string()));
1339        }
1340    }
1341    None
1342}
1343
1344/// The receipt JSON of a signed receipt: its payload's members and `sig`.
1345pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
1346    let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
1347    if let Ok(Value::Object(m)) = json::parse(r.payload()) {
1348        members.extend(m);
1349    }
1350    Value::Object(members).to_jcs()
1351}
1352
1353/// Publishes the file at `path` into `store` as the mesh `host` signing with `key` (see the
1354/// module documentation). The mesh's own namespaces are `host` alone: see [`publish_stored_in`].
1355/// The clock is the system clock, and there is no backfill. The cells are written on one lane;
1356/// see [`publish_stored_lanes`] for the parallel lanes.
1357pub fn publish_stored<S: Store + ?Sized>(
1358    store: &S,
1359    key: &SigningKey,
1360    host: &str,
1361    path: &Path,
1362) -> Result<Published, PublishError> {
1363    publish_stored_in(store, key, host, &[host.to_string()], path)
1364}
1365
1366/// As [`publish_stored`], for a mesh that owns the namespaces (hosts) `owned` (`Config::namespaces`):
1367/// a class line is accepted for a class under one of them, and refused with `E_CLASS_OWNER` (in the
1368/// [`PublishError::detail`]) under any other. Since 0.2.2. The system clock, and no backfill.
1369pub fn publish_stored_in<S: Store + ?Sized>(
1370    store: &S,
1371    key: &SigningKey,
1372    host: &str,
1373    owned: &[String],
1374    path: &Path,
1375) -> Result<Published, PublishError> {
1376    publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
1377}
1378
1379/// As [`publish_stored_in`], with the run's clock and backfill (CONTRACT-1.8b H1, rule A). The
1380/// cells are written in file order on one lane, as in 0.3.2 (CONTRACT-perf H2).
1381///
1382/// Errors: a line refused by its checks, or a binding whose start is earlier than the clock
1383/// ([`PublishErrorKind::StartBeforeNow`]), or a backfill refused ([`PublishErrorKind::Backfill`]).
1384/// Nothing is written when the first pass refuses (CONTRACT-1.7a H2).
1385pub fn publish_stored_with<S: Store + ?Sized>(
1386    store: &S,
1387    key: &SigningKey,
1388    host: &str,
1389    owned: &[String],
1390    path: &Path,
1391    opts: &PublishOptions,
1392) -> Result<Published, PublishError> {
1393    publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1394        write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1395    })
1396}
1397
1398/// As [`publish_stored_with`], with the public rule (Build 2 Part 3, D8: datumesh-http always
1399/// enforces it; `publish-stored` calls this). A file that defines a computed function (formula)
1400/// is refused unless the tests of the function (the file's, and those already held for the same
1401/// function hash) exercise every derived edge case and every branch, and all of the file's tests
1402/// pass: [`PublishErrorKind::TestsMissing`] at the function's line, [`PublishErrorKind::TestsFailing`]
1403/// at the failing test's line. Stored functions are exempt. Nothing is written when refused.
1404pub fn publish_stored_checked<S: Store + ?Sized>(
1405    store: &S,
1406    key: &SigningKey,
1407    host: &str,
1408    owned: &[String],
1409    path: &Path,
1410    opts: &PublishOptions,
1411) -> Result<Published, PublishError> {
1412    publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1413        write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1414    })
1415}
1416
1417/// As [`publish_stored_with`], with the cells written on parallel lanes (CONTRACT-perf H5, H6):
1418/// `DATUMESH_PUBLISH_LANES` (a positive integer) if set, else `min(4, available_parallelism())`.
1419/// The same receipts, versions and rows as one lane; only the order of the inserts across
1420/// different cells differs. The variable is read once per run, before the file is read.
1421pub fn publish_stored_lanes<S: Store + Sync + ?Sized>(
1422    store: &S,
1423    key: &SigningKey,
1424    host: &str,
1425    owned: &[String],
1426    path: &Path,
1427    opts: &PublishOptions,
1428) -> Result<Published, PublishError> {
1429    let lanes = publish_lanes()?;
1430    publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1431        write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1432    })
1433}
1434
1435/// The planned writes of a run as the tests see them (see [`Overlay`]).
1436fn overlay_of(ops: &[Op]) -> Result<Overlay, PublishError> {
1437    let mut overlay = Overlay::default();
1438    for op in ops {
1439        match op {
1440            Op::Function { name, start, hash, bytes, bind } => {
1441                overlay.descriptors.insert(hash.clone(), bytes.clone());
1442                if *bind {
1443                    overlay.functions.entry(name.clone()).or_default().push((start.clone(), hash.clone()));
1444                }
1445            }
1446            Op::Descriptor { hash, bytes, .. } => {
1447                overlay.descriptors.insert(hash.clone(), bytes.clone());
1448            }
1449            Op::Cell { receipt, .. } => {
1450                overlay.cell(receipt).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1451            }
1452            Op::Class { .. } | Op::Catalog { .. } => {}
1453        }
1454    }
1455    Ok(overlay)
1456}
1457
1458/// The public rule over a file (Build 2 Part 3, E5): runs the tests of each computed function the
1459/// file defines, over the store and the file's planned writes, and refuses the file when `checked`
1460/// and a test fails or the tests leave a derived case or a branch arm unexercised. Returns the
1461/// rows of the tests to hold (the file's and the held ones of those functions).
1462#[allow(clippy::too_many_arguments)]
1463pub(crate) fn check_tests<S: Store + ?Sized>(
1464    store: &S,
1465    overlay: &Overlay,
1466    tests: &[(usize, TestSpec)],
1467    formulas: &[(usize, String, String)],
1468    host: &str,
1469    now: &str,
1470    checked: bool,
1471) -> Result<Vec<RowWrite>, PublishError> {
1472    if formulas.is_empty() {
1473        return Ok(Vec::new());
1474    }
1475    let mut file_tests: Vec<(usize, TestSpec, String)> = Vec::new();
1476    for (n, spec) in tests {
1477        let at = spec.asof.clone().unwrap_or_else(|| now.to_string());
1478        match testrun::binding_of(store, overlay, &spec.function, &at) {
1479            Ok(Some((hash, _))) => file_tests.push((*n, spec.clone(), hash)),
1480            Ok(None) => {
1481                return Err(PublishError::at(
1482                    *n,
1483                    format!("a test of {}: no function of that name is bound", spec.function),
1484                ))
1485            }
1486            Err(e) => return Err(PublishError::at(*n, e)),
1487        }
1488    }
1489    let mut rows = Vec::new();
1490    for (line, name, hash) in formulas {
1491        let own: Vec<(usize, TestSpec)> =
1492            file_tests.iter().filter(|t| &t.2 == hash).map(|t| (t.0, t.1.clone())).collect();
1493        let held = testrun::held_tests(store, hash).map_err(|e| PublishError::at(*line, e))?;
1494        if !checked && own.is_empty() && held.is_empty() {
1495            continue;
1496        }
1497        let mut specs: Vec<(Option<usize>, TestSpec)> = own.into_iter().map(|(n, s)| (Some(n), s)).collect();
1498        for (_, spec) in held {
1499            if !specs.iter().any(|(_, s)| s.case == spec.case) {
1500                specs.push((None, spec));
1501            }
1502        }
1503        let ran: Vec<Ran> = specs.iter().map(|(_, s)| run_test(store, overlay, now, s)).collect();
1504        let mut run = testrun::Run::new(store, overlay, now);
1505        let built = run.built(name, hash).map_err(|e| PublishError::at(*line, format!("{name}: {}", e.0)))?;
1506        let Some(compiled) = built.compiled.as_ref() else { continue };
1507        if checked {
1508            let mut failing: Vec<(usize, String)> = Vec::new();
1509            for ((n, spec), r) in specs.iter().zip(&ran) {
1510                if let Some(n) = n {
1511                    let (result, detail) = testrun::result_of(spec, r);
1512                    if result != "pass" {
1513                        failing.push((*n, format!("{name}: test {} {detail}", spec.case)));
1514                    }
1515                }
1516            }
1517            failing.sort_by_key(|f| f.0);
1518            if let Some((n, detail)) = failing.into_iter().next() {
1519                return Err(PublishError::typed(n, PublishErrorKind::TestsFailing(detail)));
1520            }
1521            let refs: Vec<&Ran> = ran.iter().collect();
1522            let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1523            if !cov.uncovered.is_empty() || !cov.unexercised.is_empty() {
1524                let mut text = format!("{name} is missing tests:");
1525                if !cov.uncovered.is_empty() {
1526                    let cases: Vec<&str> = cov.uncovered.iter().map(|c| c.0.as_str()).collect();
1527                    text += &format!(" uncovered cases {};", cases.join(", "));
1528                }
1529                if !cov.unexercised.is_empty() {
1530                    text += &format!(" unexercised branches {}", cov.unexercised.join(", "));
1531                }
1532                return Err(PublishError::typed(
1533                    *line,
1534                    PublishErrorKind::TestsMissing(text.trim_end_matches(';').to_string()),
1535                ));
1536            }
1537        }
1538        let refs: Vec<&Ran> = ran.iter().collect();
1539        let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1540        for ((_, spec), r) in specs.iter().zip(&ran) {
1541            let (result, detail) = testrun::result_of(spec, r);
1542            rows.push(RowWrite {
1543                fn_hash: hash.clone(),
1544                case: spec.case.clone(),
1545                jcs: spec.jcs.clone(),
1546                signed_by: host.to_string(),
1547                result,
1548                detail,
1549                branches_total: cov.branches,
1550                branches_covered: cov.covered,
1551            });
1552        }
1553    }
1554    Ok(rows)
1555}
1556
1557/// As [`publish_stored_lanes`], with the public rule (see [`publish_stored_checked`]): the path of
1558/// `datumesh-mesh publish-stored` (D8).
1559pub fn publish_stored_checked_lanes<S: Store + Sync + ?Sized>(
1560    store: &S,
1561    key: &SigningKey,
1562    host: &str,
1563    owned: &[String],
1564    path: &Path,
1565    opts: &PublishOptions,
1566) -> Result<Published, PublishError> {
1567    let lanes = publish_lanes()?;
1568    publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1569        write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1570    })
1571}
1572
1573/// The first pass (the planner) and the second pass (`write_ops`) of a publish.
1574#[allow(clippy::too_many_arguments)]
1575fn publish_planned<S: Store + ?Sized>(
1576    store: &S,
1577    key: &SigningKey,
1578    host: &str,
1579    owned: &[String],
1580    path: &Path,
1581    opts: &PublishOptions,
1582    checked: bool,
1583    write_ops: impl FnOnce(&S, &[Op]) -> Result<(Published, Written), PublishError>,
1584) -> Result<Published, PublishError> {
1585    let lines = read_lines(path)?;
1586    // The key's start: the earliest `issued`.
1587    let key_from = store
1588        .namespace(host)
1589        .ok()
1590        .flatten()
1591        .and_then(|n| json::parse(&n.keydoc).ok())
1592        .and_then(|d| match d.get("keys") {
1593            Some(Value::Array(ks)) => ks
1594                .iter()
1595                .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
1596                .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
1597            _ => None,
1598        })
1599        .ok_or_else(|| {
1600            PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
1601        })?;
1602    let now = opts.clock.now();
1603    let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
1604    let mut planner = Planner {
1605        store,
1606        key,
1607        ctx,
1608        backfill: opts.backfill.is_some(),
1609        sim: Simulation::default(),
1610        reads: Reads::default(),
1611        ops: Vec::new(),
1612        binds: Vec::new(),
1613        tests: Vec::new(),
1614        formulas: Vec::new(),
1615    };
1616    planner.run(&lines)?;
1617    let Planner { ops, binds, tests, formulas, .. } = planner;
1618    if opts.backfill.is_some() {
1619        backfill_check(store, &binds)?;
1620    }
1621    // The tests of the file's computed functions, run over the store and the file's own writes:
1622    // refused before anything is written (Build 2 Part 3, D8).
1623    let rows = check_tests(store, &overlay_of(&ops)?, &tests, &formulas, host, &now, checked)?;
1624    let (out, written) = write_ops(store, &ops)?;
1625    index_cell_values(store, &ops).map_err(|e| PublishError::at(0, format!("catalogue: {e}")))?;
1626    testrun::apply_rows(store, &rows, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1627    let rebound: Vec<String> = ops
1628        .iter()
1629        .filter_map(|op| match op {
1630            Op::Function { name, bind: true, .. } => Some(name.clone()),
1631            _ => None,
1632        })
1633        .collect();
1634    testrun::rerun_readers(store, &rebound, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1635    if let Some(log) = &opts.backfill {
1636        if out.functions + out.classes + out.versions > 0 {
1637            append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
1638        }
1639    }
1640    Ok(out)
1641}