Skip to main content

datumesh_http/
publish.rs

1//! Publishing stored cells as the mesh (`datumesh-mesh publish-stored`): a JSON lines file of
2//! stored functions and their cells, signed with the mesh's key, put in the store.
3//!
4//! Each line is one of:
5//!
6//! * a descriptor (since 0.3.1, CONTRACT-1.7a H1): `{"descriptor": <function descriptor>}`: the
7//!   descriptor is put with `put_descriptor` and binds nothing. Only a stored function is
8//!   accepted (`code.lang` = `stored`, `code.body` = `""`, `inputs` = `[]`); any other is refused
9//!   at its line. A class later in the file (or in a later run) may name its hash as a method;
10//! * a function: `{"function": "<ns>/<name>", "signature": {...}, "bound": "<instant>"}`: the
11//!   descriptor `{"code": {"body": "", "lang": "stored"}, "inputs": [], "signature": ...}`
12//!   is put and the name bound to it from `bound` (unless it already is);
13//! * a formula function: `{"formula": "<ns>/<name>", "source": "<formula>", "signature":
14//!   {...}, "bound": "<instant>"}`: defined with `datumesh_formula::definition::define_function`
15//!   (its callees as bound at `bound`) and published (`put_descriptor`, `bind_name` from
16//!   `bound`) unless the name is already bound to that function; its cells are evaluated on
17//!   demand by the mesh;
18//! * a class (since 0.2.2, spec-0.5.0, 16.2): `{"class": "<ns>/<name>", "descriptor": {...as
19//!   written...}, "bound": "<instant>"}`: defined with the formula crate's `define_class` (core's
20//!   rules: the class is under one of the mesh's own namespaces, else `E_CLASS_OWNER`; the method
21//!   functions are those bound as of `bound`), `put_class`, and `bind_class` from `bound` unless the
22//!   name is already bound to that class hash. Method functions come first in the file (or in an
23//!   earlier one), `bound` later than theirs. A line with both `class` and `descriptor` is a class
24//!   line;
25//! * a cell: `{"cell": {"fn": "<ns>/<name>", "args": {<name>: <typed scalar>}}, "value": <typed
26//!   scalar> or {"blob": "<file>", "media_type": "<type>"}, "tx"?, "asof"?, "issued"?, "licence",
27//!   "summary"}`: a blob's file (relative to the JSON lines file) is put with `put_blob` and the
28//!   value is `<sha256>:<length>:<media type>`; the receipt is signed with the mesh's key
29//!   (`computed_by` the mesh host, `publisher` the cell's namespace, `valid` its valid-time
30//!   arguments; `issued` defaults to the later of `asof` and the key's start) and stored with
31//!   `put_receipt` and `put_version`. `tx` and `asof` default to the run's clock now (since
32//!   0.3.1, H4), each independently. A `source` object (since 0.4.0, spec-0.6.0 C2: `name`, `url`,
33//!   `retrieved`, and optionally `sha256` and `takeover`, nothing else) marks the value as relayed: the
34//!   receipt signs it, and its trust is `relayed`; its rules are the receipt's (`retrieved` no later
35//!   than `issued`, a publisher that does not relay itself, `https://` URLs), refused at the line.
36//!
37//! The function of a cell is the one its name is bound to as of the cell's `tx`. Re-running a
38//! file changes nothing (a cell line without `tx` whose latest version states the same thing is
39//! skipped, since 0.3.1): descriptors and blobs are content-addressed, signing is deterministic
40//! (the same receipt, the same hash), and a version already there is skipped. The key is never
41//! logged.
42//!
43//! Rule A (since 0.3.0, CONTRACT-1.8b H1): no binding starts earlier than the mesh clock. The
44//! start of a binding (a name's, a class's, or a class method's `N/m`) is its `bound`, or the
45//! clock's now when `bound` is left out, and a start earlier than now is refused with
46//! [`PublishErrorKind::StartBeforeNow`]. A `--backfill` run ([`PublishOptions::backfill`]) lifts
47//! the start check, and is refused ([`PublishErrorKind::Backfill`]) when a name it would bind has
48//! a binding, or when a namespace of such a name has any binding (store 0.4.2's
49//! `has_bindings_under`). A run that writes appends one line to the log: `{"at": …, "namespaces":
50//! […], "starts": […]}`, a JCS object.
51//!
52//! No partial writes (since 0.3.1, CONTRACT-1.7a H2): the file is read in two passes. The first
53//! plans every write the second one makes, and checks each as the store will: a bind is refused
54//! when its start is not later than the latest binding of its name (the store's, or one planned
55//! earlier in the file, W1b), when the name is of the other kind (a class or a function), or when
56//! its namespace is suspended; a cell's function must be bound as of its `tx` (counting the
57//! earlier binds of the file), its blob must be readable, `tx <= asof <= issued`, and a version
58//! at or after its `tx` must be this receipt. Receipts are signed in the first pass (signing is
59//! deterministic), so their hashes are known. A file that the first pass refuses writes nothing.
60
61use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
62use std::io::{BufRead, Write};
63use std::path::{Path, PathBuf};
64use std::sync::atomic::{AtomicBool, Ordering};
65use std::sync::Arc;
66
67use datumesh_core::json::{self, Value};
68use datumesh_core::{
69    sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey,
70    Source, Type,
71};
72use datumesh_engine::{Clock, SystemClock};
73use datumesh_formula::definition::{Callee, Callees, ClassCallee};
74use datumesh_store::{Binding, CellWrite, FunctionTest, Namespace, Store};
75
76use crate::testrun::{self, run_test, Overlay, Ran, RowWrite, TestSpec};
77
78use crate::config::ConfigError;
79
80/// The end of time: a binding as of it is the latest one.
81const END: &str = "9999-12-31T23:59:59.999999Z";
82
83/// Cell lines planned together, and cell writes per `put_cells` call (CONTRACT-perf H1, H2).
84const CHUNK: usize = 1_000;
85
86/// The environment variable of the publish lanes (CONTRACT-perf H5), read once per run.
87const LANES_VAR: &str = "DATUMESH_PUBLISH_LANES";
88
89/// The lanes of a run (CONTRACT-perf H5): `DATUMESH_PUBLISH_LANES` (a positive integer), else
90/// `min(4, available_parallelism)`, at least 1. An invalid value refuses the run before any write.
91fn publish_lanes() -> Result<usize, PublishError> {
92    let bad = |detail: String| PublishError::at(0, ConfigError::new(LANES_VAR, detail).to_string());
93    match std::env::var(LANES_VAR) {
94        Err(std::env::VarError::NotPresent) => Ok(std::thread::available_parallelism().map_or(1, |n| n.get()).min(4)),
95        Ok(v) => match v.parse::<usize>() {
96            Ok(n) if n > 0 => Ok(n),
97            _ => Err(bad(format!("must be a positive integer, not {v:?}"))),
98        },
99        Err(e) => Err(bad(e.to_string())),
100    }
101}
102
103/// What a run put.
104#[non_exhaustive]
105#[derive(Clone, Debug, Default, PartialEq, Eq)]
106pub struct Published {
107    /// Functions newly bound (stored and formula).
108    pub functions: usize,
109    /// Classes newly bound (since 0.2.2).
110    pub classes: usize,
111    /// Cell lines read.
112    pub cells: usize,
113    /// Receipts stored (new or already there).
114    pub receipts: usize,
115    /// Versions newly added.
116    pub versions: usize,
117    /// Blobs put (new or already there).
118    pub blobs: usize,
119    /// Blob bytes put.
120    pub blob_bytes: u64,
121    /// Descriptors newly put by descriptor lines (since 0.3.1); a re-run counts 0.
122    pub descriptors: u64,
123}
124
125/// How a run reads the clock and whether it backfills (CONTRACT-1.8b H1).
126#[derive(Clone)]
127pub struct PublishOptions {
128    /// The mesh clock: the start of a binding left out is its now, and a start earlier than it is
129    /// refused (rule A).
130    pub clock: Arc<dyn Clock + Send + Sync>,
131    /// The operator log of a `--backfill` run (`DATUMESH_BACKFILL_LOG`). `Some` lifts the start
132    /// check, for a file whose names and namespaces have no bindings yet, and appends a line to
133    /// the log for each run that writes.
134    pub backfill: Option<PathBuf>,
135}
136
137impl PublishOptions {
138    /// The system clock, and no backfill.
139    pub fn system() -> PublishOptions {
140        PublishOptions { clock: Arc::new(SystemClock), backfill: None }
141    }
142}
143
144/// Why a line (or the file, at line 0) was refused.
145#[non_exhaustive]
146#[derive(Clone, Debug, PartialEq, Eq)]
147pub enum PublishErrorKind {
148    /// Any other refusal: [`PublishError::detail`] says what.
149    Line,
150    /// A binding's start is earlier than the mesh clock (rule A).
151    StartBeforeNow {
152        /// The name (or class name, or method name) that would bind.
153        name: String,
154        /// Its start.
155        start: String,
156        /// The mesh clock.
157        now: String,
158    },
159    /// A `--backfill` run that would write over a binding, or under a namespace with one.
160    Backfill(String),
161    /// `import-vectors`: a name the vectors would bind is under a namespace the mesh owns.
162    OwnedVectorName(String),
163    /// The public rule (D8, Build 2 Part 3, `publish_stored_checked`): a computed function's tests
164    /// do not cover the derived cases or every branch. The text lists what is missing.
165    TestsMissing(String),
166    /// The public rule: a test of a computed function fails. The text names the case and the
167    /// numbers.
168    TestsFailing(String),
169}
170
171impl std::fmt::Display for PublishErrorKind {
172    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
173        match self {
174            PublishErrorKind::Line => f.write_str("refused"),
175            PublishErrorKind::StartBeforeNow { name, start, now } => {
176                write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
177            }
178            PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
179            PublishErrorKind::OwnedVectorName(name) => {
180                write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
181            }
182            PublishErrorKind::TestsMissing(text) => write!(f, "E_TESTS_MISSING: {text}"),
183            PublishErrorKind::TestsFailing(text) => write!(f, "E_TESTS_FAILING: {text}"),
184        }
185    }
186}
187
188/// Why a run stopped: the line, what was wrong, and its typed kind.
189#[derive(Clone, Debug, PartialEq, Eq)]
190pub struct PublishError {
191    /// The line number (1-based; 0 for the file).
192    pub line: usize,
193    /// Why, in words.
194    pub detail: String,
195    /// Why, typed.
196    pub kind: PublishErrorKind,
197}
198
199impl PublishError {
200    fn at(line: usize, detail: impl Into<String>) -> PublishError {
201        PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
202    }
203
204    pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
205        PublishError { line, detail: kind.to_string(), kind }
206    }
207}
208
209impl std::fmt::Display for PublishError {
210    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
211        write!(f, "line {}: {}", self.line, self.detail)
212    }
213}
214
215impl std::error::Error for PublishError {}
216
217fn s(v: &str) -> Value {
218    Value::String(v.to_string())
219}
220
221/// The namespace of a name: the name minus its last `/` segment.
222fn namespace_of(name: &str) -> &str {
223    name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
224}
225
226/// The host of a name: its first segment.
227fn host_of(name: &str) -> &str {
228    name.split('/').next().unwrap_or("")
229}
230
231/// The definer of a class `name`: its own namespace when the mesh owns the name's host, else the
232/// mesh's host (which no class name under another host has, so core answers `E_CLASS_OWNER`).
233fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
234    let host_of_name = name.split('/').next().unwrap_or("");
235    if owned.iter().any(|o| o == host_of_name) {
236        namespace_of(name)
237    } else {
238        host
239    }
240}
241
242/// The start of a line's binding: its `bound`, or the clock's now when it is left out.
243fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
244    match v.get("bound") {
245        None => Ok(now.to_string()),
246        Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
247    }
248}
249
250/// The bindings and versions a pass one has planned so far: the file's own, which are seen before
251/// the store's.
252#[derive(Default)]
253struct Simulation {
254    /// Function name to the (tx, function hash) of each bind the file would make.
255    functions: BTreeMap<String, Vec<(String, String)>>,
256    /// Class name to the (tx, class hash) of each class bind the file would make.
257    classes: BTreeMap<String, Vec<(String, String)>>,
258    /// Canonical bytes by function hash, of the descriptors the file would store.
259    descriptors: BTreeMap<String, Vec<u8>>,
260    /// Canonical bytes by class hash, of the classes the file would store.
261    class_bytes: BTreeMap<String, Vec<u8>>,
262    /// Cell id to the (tx, receipt hash) of each version the file would add, in tx order.
263    versions: BTreeMap<String, Vec<(String, String)>>,
264}
265
266impl Simulation {
267    /// The latest of `binds`' entries for `name` with tx <= `at`.
268    fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
269        binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
270    }
271}
272
273/// The callees of a definition: the store's bindings, and with `sim` the file's own over them.
274struct Callers<'a, S: Store + ?Sized> {
275    store: &'a S,
276    sim: Option<&'a Simulation>,
277}
278
279impl<S: Store + ?Sized> Callers<'_, S> {
280    fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
281        if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
282            return Ok(Some(b.clone()));
283        }
284        Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
285    }
286
287    fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
288        if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
289            return Ok(Some(b.clone()));
290        }
291        self.store.class(hash).map_err(|e| e.to_string())
292    }
293}
294
295impl<S: Store + ?Sized> Callees for Callers<'_, S> {
296    fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
297        let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
298        let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
299        let (fn_hash, start) = match (stored, simmed) {
300            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
301            (Some(b), _) => (b.fn_hash, b.tx),
302            (None, Some((t, h))) => (h, t),
303            (None, None) => return Ok(None),
304        };
305        let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
306        let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
307        Ok(Some(Callee { fn_hash, start, descriptor }))
308    }
309
310    fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
311        let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
312        let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
313        let (class_hash, start) = match (stored, simmed) {
314            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
315            (Some(b), _) => (b.class_hash, b.tx),
316            (None, Some((t, h))) => (h, t),
317            (None, None) => return Ok(None),
318        };
319        let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
320        let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
321        Ok(Some(ClassCallee { class_hash, start, descriptor }))
322    }
323
324    fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
325        let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
326        ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
327    }
328
329    fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
330        let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
331        json::parse(&bytes).map(Some).map_err(|e| e.to_string())
332    }
333}
334
335/// What a run reads from the store again and again, kept for the run (CONTRACT-perf H1). The
336/// store does not change during the first pass, so each answer is the same whenever it is asked,
337/// and a failed read is kept as its message, which is what the check that asks it reports. A
338/// prefetch that fails keeps nothing: each line then asks for itself and fails at its own line.
339#[derive(Default)]
340struct Reads {
341    bindings: HashMap<(String, String), Result<Option<Binding>, String>>,
342    descriptors: HashMap<String, Option<Vec<u8>>>,
343    namespaces: HashMap<String, Result<Option<Namespace>, String>>,
344    /// The store's latest version of a cell (tx, receipt hash), prefetched per chunk.
345    latest_of: HashMap<String, Option<(String, String)>>,
346    /// The store's receipt, prefetched for the latest receipts a skip check reads.
347    receipt_of: HashMap<String, Option<Vec<u8>>>,
348}
349
350impl Reads {
351    fn binding<S: Store + ?Sized>(&mut self, store: &S, name: &str, at: &str) -> Result<Option<Binding>, String> {
352        let key = (name.to_string(), at.to_string());
353        if let Some(r) = self.bindings.get(&key) {
354            return r.clone();
355        }
356        let r = store.resolve_binding(name, at).map_err(|e| e.to_string());
357        self.bindings.insert(key, r.clone());
358        r
359    }
360
361    fn descriptor<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
362        if let Some(b) = self.descriptors.get(hash) {
363            return b.clone();
364        }
365        let b = store.descriptor(hash).map(|b| b.into_owned());
366        self.descriptors.insert(hash.to_string(), b.clone());
367        b
368    }
369
370    fn namespace<S: Store + ?Sized>(&mut self, store: &S, host: &str) -> Result<Option<Namespace>, String> {
371        if let Some(r) = self.namespaces.get(host) {
372            return r.clone();
373        }
374        let r = store.namespace(host).map_err(|e| e.to_string());
375        self.namespaces.insert(host.to_string(), r.clone());
376        r
377    }
378
379    /// The store's latest version of `id`: prefetched, else asked now.
380    fn latest<S: Store + ?Sized>(&mut self, store: &S, id: &str) -> Result<Option<(String, String)>, String> {
381        if let Some(l) = self.latest_of.get(id) {
382            return Ok(l.clone());
383        }
384        store.latest(id).map(|v| v.map(|v| (v.tx, v.receipt_hash))).map_err(|e| e.to_string())
385    }
386
387    /// The stored receipt `hash`, as `receipt` answers: prefetched, else asked now.
388    fn receipt<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
389        if let Some(b) = self.receipt_of.get(hash) {
390            return b.clone();
391        }
392        store.receipt(hash).map(|b| b.into_owned())
393    }
394
395    /// One `latest_many` for the ids not read yet.
396    fn prefetch_latest<S: Store + ?Sized>(&mut self, store: &S, ids: &[&str]) {
397        let mut seen = HashSet::new();
398        let missing: Vec<&str> =
399            ids.iter().copied().filter(|id| !self.latest_of.contains_key(*id) && seen.insert(*id)).collect();
400        if missing.is_empty() {
401            return;
402        }
403        if let Ok(found) = store.latest_many(&missing) {
404            for (id, v) in missing.iter().zip(found) {
405                self.latest_of.insert(id.to_string(), v.map(|v| (v.tx, v.receipt_hash)));
406            }
407        }
408    }
409
410    /// One `receipts_many` for the hashes not read yet.
411    fn prefetch_receipts<S: Store + ?Sized>(&mut self, store: &S, hashes: &[&str]) {
412        let mut seen = HashSet::new();
413        let missing: Vec<&str> =
414            hashes.iter().copied().filter(|h| !self.receipt_of.contains_key(*h) && seen.insert(*h)).collect();
415        if missing.is_empty() {
416            return;
417        }
418        if let Ok(found) = store.receipts_many(&missing) {
419            for (h, b) in missing.iter().zip(found) {
420                self.receipt_of.insert(h.to_string(), b);
421            }
422        }
423    }
424}
425
426/// `f` over `items` on up to `available_parallelism()` threads; the results in item order.
427fn parallel_owned<T: Send, R: Send>(items: Vec<T>, f: impl Fn(T) -> R + Sync) -> Vec<R> {
428    let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(items.len());
429    if threads <= 1 {
430        return items.into_iter().map(f).collect();
431    }
432    let size = items.len().div_ceil(threads);
433    let mut parts: Vec<Vec<T>> = Vec::with_capacity(threads);
434    let mut rest = items.into_iter();
435    loop {
436        let part: Vec<T> = rest.by_ref().take(size).collect();
437        if part.is_empty() {
438            break;
439        }
440        parts.push(part);
441    }
442    std::thread::scope(|s| {
443        let handles: Vec<_> = parts
444            .into_iter()
445            .map(|part| {
446                let f = &f;
447                s.spawn(move || part.into_iter().map(f).collect::<Vec<R>>())
448            })
449            .collect();
450        handles.into_iter().flat_map(|h| h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect()
451    })
452}
453
454/// The function hash `name` means at the end of time: the file's own over the store's.
455fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
456    if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
457        return Ok(Some(h));
458    }
459    store.resolve_name(name, END).map_err(|e| e.to_string())
460}
461
462/// The class hash `name` means at the end of time: the file's own over the store's.
463fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
464    if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
465        return Ok(Some(h));
466    }
467    store.resolve_class(name, END).map_err(|e| e.to_string())
468}
469
470/// The function hash `name` means as of `at`, counting the file's earlier binds.
471fn function_at<S: Store + ?Sized>(
472    reads: &mut Reads,
473    store: &S,
474    sim: &Simulation,
475    name: &str,
476    at: &str,
477) -> Result<Option<String>, String> {
478    let stored = reads.binding(store, name, at)?;
479    let simmed = Simulation::latest(&sim.functions, name, at);
480    Ok(match (stored, simmed) {
481        (Some(b), Some((t, h))) if t >= b.tx => Some(h),
482        (Some(b), _) => Some(b.fn_hash),
483        (None, Some((_, h))) => Some(h),
484        (None, None) => None,
485    })
486}
487
488/// The latest binding tx of `name` (a class's, or a function's), the store's or the file's.
489fn latest_bind<S: Store + ?Sized>(
490    store: &S,
491    sim: &Simulation,
492    name: &str,
493    class: bool,
494) -> Result<Option<String>, String> {
495    let stored: Option<String> = if class {
496        store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
497    } else {
498        store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
499    };
500    let binds = if class { &sim.classes } else { &sim.functions };
501    let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
502    Ok(match (stored, planned) {
503        (Some(a), Some(b)) => Some(a.max(b)),
504        (a, b) => a.or(b),
505    })
506}
507
508/// Whether `name` is bound as the other kind: a function for a class, a class for a function.
509fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
510    if class {
511        Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
512    } else {
513        Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
514            || sim.classes.contains_key(name))
515    }
516}
517
518/// The latest version of a cell, the store's or the file's: its tx and receipt hash.
519fn cell_latest<S: Store + ?Sized>(
520    reads: &mut Reads,
521    store: &S,
522    sim: &Simulation,
523    id: &str,
524) -> Result<Option<(String, String)>, String> {
525    let stored = reads.latest(store, id)?;
526    let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
527    Ok(match (stored, planned) {
528        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
529        (a, b) => a.or(b),
530    })
531}
532
533/// The receipt hash of a cell's version as of `at`, the store's or the file's.
534fn cell_version_asof<S: Store + ?Sized>(
535    store: &S,
536    sim: &Simulation,
537    id: &str,
538    at: &str,
539) -> Result<Option<String>, String> {
540    let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
541    let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
542    let latest = match (stored, planned) {
543        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
544        (a, b) => a.or(b),
545    };
546    Ok(latest.map(|x| x.1))
547}
548
549/// The descriptor bytes of a function hash: the file's own, else the store's (memoized).
550fn function_bytes_of<S: Store + ?Sized>(
551    reads: &mut Reads,
552    store: &S,
553    sim: &Simulation,
554    hash: &str,
555) -> Result<Option<Vec<u8>>, String> {
556    if let Some(b) = sim.descriptors.get(hash) {
557        return Ok(Some(b.clone()));
558    }
559    Ok(reads.descriptor(store, hash))
560}
561
562/// Whether a namespace is suspended (the store refuses binds and class puts under it).
563fn suspended<S: Store + ?Sized>(reads: &mut Reads, store: &S, host: &str, line: usize) -> Result<(), PublishError> {
564    match reads.namespace(store, host) {
565        Ok(Some(n)) if n.suspended_at.is_some() => {
566            Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
567        }
568        Ok(_) => Ok(()),
569        Err(e) => Err(PublishError::at(line, e)),
570    }
571}
572
573/// A binding a run would make: the line that makes it and its name.
574struct Bind {
575    line: usize,
576    name: String,
577}
578
579/// What a run writes, for the backfill log: the namespaces and starts of its binds.
580#[derive(Default)]
581struct Written {
582    namespaces: BTreeSet<String>,
583    starts: BTreeSet<String>,
584}
585
586impl Written {
587    fn bound(&mut self, name: &str, start: &str) {
588        self.namespaces.insert(namespace_of(name).to_string());
589        self.starts.insert(start.to_string());
590    }
591}
592
593/// The per-run values both passes use.
594struct Ctx<'a> {
595    host: &'a str,
596    owned: &'a [String],
597    /// The start of the mesh's key.
598    key_from: &'a str,
599    /// The clock's now, read once for the run.
600    now: &'a str,
601    /// The directory of the file (blobs are relative to it).
602    base: &'a Path,
603}
604
605/// One write the second pass makes, planned (and checked) by the first.
606enum Op {
607    /// A descriptor line: put; counted when newly put.
608    Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
609    /// A function (stored or formula): put its descriptor, and bind the name when `bind`.
610    Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
611    /// A class: put it, and bind it (its methods with it, in the store) when `bind`.
612    Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
613    /// A cell: its blob, its receipt, and the version when `version`.
614    Cell { tx: String, id: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
615    /// A catalogue line (R5): the entry, and its dictionaries (`publisher`, signed by the run's key).
616    Catalog {
617        entry: datumesh_store::CatalogEntry,
618        dictionaries: Vec<(String, Vec<datumesh_store::KeyValue>)>,
619        open_keys: Vec<String>,
620    },
621}
622
623/// The lines of a file, numbered, without the blank ones.
624fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
625    let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
626    let mut out = Vec::new();
627    for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
628        let n = i + 1;
629        let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
630        if !line.trim().is_empty() {
631            out.push((n, line));
632        }
633    }
634    Ok(out)
635}
636
637/// The dictionary values the file's cells have for the string arguments (CONTRACT-build2-part2-read
638/// R3): each distinct (function, key, value) that no dictionary has yet is stored with an empty label
639/// (the value stands for itself). A dictionary's label is never replaced by a cell's.
640fn index_cell_values<S: Store + ?Sized>(store: &S, ops: &[Op]) -> Result<(), String> {
641    let mut seen = BTreeSet::new();
642    for op in ops {
643        if let Op::Cell { receipt, .. } = op {
644            if let Ok(r) = datumesh_core::Receipt::from_json(receipt) {
645                cell_string_values(&r, &mut seen);
646            }
647        }
648    }
649    store_cell_values(store, seen)
650}
651
652/// The (function, key, value) of each string argument of a receipt's cell, into `out`.
653pub(crate) fn cell_string_values(r: &datumesh_core::Receipt, out: &mut BTreeSet<(String, String, String)>) {
654    let cell = r.cell();
655    for (key, arg) in &cell.args {
656        if let datumesh_core::Arg::Value(datumesh_core::Scalar::String(v)) = arg {
657            if !v.is_empty() && v.len() <= 1024 {
658                out.insert((cell.function_name(), key.clone(), v.clone()));
659            }
660        }
661    }
662}
663
664/// The most distinct values a string key of a function takes from its cells before the key is open
665/// (`DATUMESH_DICTIONARY_MAX`, CONTRACT-0.4.9 H7): read from the process environment when it is needed; 1000 when
666/// unset or not a positive integer.
667pub(crate) fn dictionary_limit() -> u64 {
668    std::env::var("DATUMESH_DICTIONARY_MAX")
669        .ok()
670        .and_then(|v| v.trim().parse::<u64>().ok())
671        .filter(|n| *n >= 1)
672        .unwrap_or(1000)
673}
674
675/// Stores the values of [`cell_string_values`] that no dictionary has yet (R3), by the rule of CONTRACT-0.4.9 H7:
676/// for each (function, key) the new values are added only while the key would have at most
677/// [`dictionary_limit`] values; a publish that would take it past the limit adds none of them and marks the key
678/// open. An open key takes no automatic values again; the values it has stay.
679pub(crate) fn store_cell_values<S: Store + ?Sized>(
680    store: &S,
681    seen: BTreeSet<(String, String, String)>,
682) -> Result<(), String> {
683    let mut groups: BTreeMap<(String, String), Vec<String>> = BTreeMap::new();
684    for (owner, key, value) in seen {
685        groups.entry((owner, key)).or_default().push(value);
686    }
687    let limit = dictionary_limit();
688    for ((owner, key), values) in groups {
689        if store.key_is_open(&owner, &key).map_err(|e| e.to_string())? {
690            continue;
691        }
692        let fresh: Vec<String> =
693            values.into_iter().filter(|v| !crate::find::has_value(store, &owner, &key, v)).collect();
694        if fresh.is_empty() {
695            continue;
696        }
697        let have = store.key_value_count(&owner, &key).map_err(|e| e.to_string())?;
698        if have + fresh.len() as u64 > limit {
699            store.mark_key_open(&owner, &key).map_err(|e| e.to_string())?;
700            continue;
701        }
702        let kvs: Vec<datumesh_store::KeyValue> = fresh
703            .into_iter()
704            .map(|value| datumesh_store::KeyValue { value, label: String::new(), aliases: Vec::new() })
705            .collect();
706        store.upsert_key_values(&owner, &key, &kvs).map_err(|e| e.to_string())?;
707    }
708    Ok(())
709}
710
711/// The function a test row is named by when it is written (CONTRACT-0.4.9 H1): a file's own test names the
712/// function its `test` member names; a held test is the group's function.
713fn row_name(own: Option<usize>, spec: &TestSpec, group: &str) -> String {
714    match own {
715        Some(_) => spec.function.clone(),
716        None => group.to_string(),
717    }
718}
719
720/// Whether a line is a cell line: none of the other kinds (the dispatch order of `plan_line`).
721fn is_cell_line(v: &Value) -> bool {
722    v.get("class").and_then(Value::as_str).is_none()
723        && v.get("descriptor").is_none()
724        && v.get("function").and_then(Value::as_str).is_none()
725        && v.get("formula").and_then(Value::as_str).is_none()
726        && v.get("cell").is_some()
727}
728
729/// A cell line prepared for signing: everything its receipt takes, checked up to there.
730struct Pending {
731    line: usize,
732    ns: String,
733    cell: Cell,
734    value: Scalar,
735    valid: Args,
736    tx: String,
737    asof: String,
738    issued: String,
739    has_tx: bool,
740    licence: String,
741    summary: String,
742    blob: Option<Vec<u8>>,
743    /// The `source` of a relayed value (spec-0.6.0, 8.1), as the line gives it.
744    source: Option<Source>,
745}
746
747/// The `source` object of a cell line (spec-0.6.0, C2): exactly `name`, `url` and `retrieved`, with
748/// `sha256` and `takeover` optional, all strings. Its rules (a DNS name, an instant, an `https://`
749/// takeover, a publisher that does not relay itself) are the receipt's, checked when it is signed.
750fn source_of_line(v: &Value) -> Result<Source, String> {
751    let Value::Object(m) = v else { return Err("source is an object".into()) };
752    for (k, x) in m {
753        if !matches!(k.as_str(), "name" | "url" | "retrieved" | "sha256" | "takeover") {
754            return Err(format!("source has no member {k}"));
755        }
756        if !matches!(x, Value::String(_)) {
757            return Err(format!("source.{k} is a string"));
758        }
759    }
760    let text = |k: &str| -> Result<&str, String> {
761        v.get(k).and_then(Value::as_str).ok_or_else(|| format!("source has no {k}"))
762    };
763    let mut out = Source::new(text("name")?, text("url")?, text("retrieved")?);
764    if let Some(h) = v.get("sha256").and_then(Value::as_str) {
765        out = out.with_sha256(h);
766    }
767    if let Some(t) = v.get("takeover").and_then(Value::as_str) {
768        out = out.with_takeover(t);
769    }
770    Ok(out)
771}
772
773impl Pending {
774    /// Signs the receipt (deterministic) and takes its hash and cell id. Runs on any thread.
775    fn sign(self, key: &SigningKey, host: &str) -> Result<Signed, PublishError> {
776        let line = self.line;
777        let err = move |d: String| PublishError::at(line, d);
778        let b = ReceiptBuilder::new()
779            .cell(self.cell.clone())
780            .value(self.value)
781            .valid(self.valid)
782            .asof(self.asof.as_str())
783            .tx(self.tx.as_str())
784            .issued(self.issued.as_str())
785            .publisher(self.ns.as_str())
786            .signer_host(host)
787            .licence(self.licence.as_str())
788            .summary(self.summary.as_str());
789        let b = match self.source {
790            Some(s) => b.source(s),
791            None => b,
792        };
793        let r = b.sign(key).map_err(|e| err(e.to_string()))?;
794        let receipt = receipt_json(&r);
795        let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
796        let id = self.cell.id();
797        Ok(Signed { line, has_tx: self.has_tx, tx: self.tx, id, hash, receipt, blob: self.blob })
798    }
799}
800
801/// A cell line signed: what its checks and its write take.
802struct Signed {
803    line: usize,
804    has_tx: bool,
805    tx: String,
806    id: String,
807    hash: String,
808    receipt: Vec<u8>,
809    blob: Option<Vec<u8>>,
810}
811
812/// Pass one: plans the writes of the lines and checks each as the store will (CONTRACT-1.7a H2).
813struct Planner<'a, S: Store + ?Sized> {
814    store: &'a S,
815    key: &'a SigningKey,
816    ctx: Ctx<'a>,
817    backfill: bool,
818    sim: Simulation,
819    reads: Reads,
820    ops: Vec<Op>,
821    binds: Vec<Bind>,
822    /// The test lines of the file, with their lines (Build 2 Part 3).
823    tests: Vec<(usize, TestSpec)>,
824    /// The computed functions the file defines: (line, name, function hash).
825    formulas: Vec<(usize, String, String)>,
826}
827
828impl<S: Store + ?Sized> Planner<'_, S> {
829    fn callers(&self) -> Callers<'_, S> {
830        Callers { store: self.store, sim: Some(&self.sim) }
831    }
832
833    /// Plans the lines in file order. Cell lines are planned in chunks of up to `CHUNK`
834    /// consecutive ones; any other line ends a chunk (CONTRACT-perf H1).
835    fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
836        let mut cells: Vec<(usize, Value)> = Vec::new();
837        for (n, text) in lines {
838            let n = *n;
839            match json::parse(text.as_bytes()).map_err(|e| PublishError::at(n, e.to_string())) {
840                Ok(v) if is_cell_line(&v) => {
841                    // The reserved paths (CONTRACT-0.4.9 H8), as for any other line.
842                    if let Some(name) = v.get("cell").and_then(|c| c.get("fn")).and_then(Value::as_str) {
843                        crate::reserved::check(self.ctx.host, name, None).map_err(|e| PublishError::at(n, e))?;
844                    }
845                    cells.push((n, v));
846                    if cells.len() == CHUNK {
847                        self.plan_cells(std::mem::take(&mut cells))?;
848                    }
849                }
850                other => {
851                    // The cells before this line are checked first: a failure of either is the
852                    // file's first.
853                    self.plan_cells(std::mem::take(&mut cells))?;
854                    self.plan_line(n, &other?)?;
855                }
856            }
857        }
858        self.plan_cells(cells)
859    }
860
861    /// One line that is not a cell (the dispatch of the lines before cells, as it was).
862    fn plan_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
863        let err = |d: String| PublishError::at(n, d);
864        // The reserved paths (CONTRACT-0.4.9 H8): a name a line gives is checked before anything else of it. The
865        // file is written by the mesh's own key, which takes no key namespace's name.
866        let named =
867            v.get("class").or_else(|| v.get("function")).or_else(|| v.get("formula")).or_else(|| v.get("catalog"));
868        if let Some(name) = named.and_then(Value::as_str) {
869            crate::reserved::check(self.ctx.host, name, None).map_err(err)?;
870        }
871        if let Some(name) = v.get("class").and_then(Value::as_str) {
872            self.class_line(n, name, v)?;
873        } else if let Some(d) = v.get("descriptor") {
874            self.descriptor_line(n, d)?;
875        } else if let Some(name) = v.get("function").and_then(Value::as_str) {
876            let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
877            let start = bound_of(v, n, self.ctx.now)?;
878            let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
879            self.function(n, name, start, fd.hash(), fd.canonical())?;
880        } else if v.get("catalog").is_some() {
881            self.catalog_line(n, v)?;
882        } else if let Some(name) = v.get("formula").and_then(Value::as_str) {
883            let source =
884                v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
885            let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
886            let start = bound_of(v, n, self.ctx.now)?;
887            let def = datumesh_formula::definition::define_function(
888                name,
889                source,
890                sig,
891                &Default::default(),
892                &self.callers(),
893                &start,
894            )
895            .map_err(|e| err(format!("define {name}: {e}")))?;
896            let hash = def.fn_hash.clone();
897            self.function(n, name, start, def.fn_hash, def.bytes)?;
898            self.formulas.push((n, name.to_string(), hash));
899        } else if v.get("test").is_some() {
900            let spec = testrun::parse_test(v).map_err(err)?;
901            self.tests.push((n, spec));
902        } else {
903            return Err(err("neither a function nor a cell".into()));
904        }
905        Ok(())
906    }
907
908    /// `{"catalog": <name>, "kind", "label", "description", "args", "examples", "dictionary"}` (R5):
909    /// the publisher's text for a function or class bound in this file or the store, under a host
910    /// this mesh owns. Checked here, so a refused line writes nothing.
911    fn catalog_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
912        let err = |d: String| PublishError::at(n, d);
913        let d = crate::catalog::parse(v, "catalog").map_err(err)?;
914        let host = host_of(&d.name);
915        if host != self.ctx.host && !self.ctx.owned.iter().any(|o| o == host) {
916            return Err(err(format!("{} is not under a host this mesh owns", d.name)));
917        }
918        let hash = match d.kind {
919            datumesh_store::CatalogKind::Class => current_class(self.store, &self.sim, &d.name).map_err(err)?,
920            _ => current_function(self.store, &self.sim, &d.name).map_err(err)?,
921        };
922        if hash.is_none() {
923            return Err(err(format!("{} is bound nowhere", d.name)));
924        }
925        let entry = d.entry(hash, datumesh_store::CatalogSource::Publisher, Some(self.key.key_id().to_string()));
926        self.ops.push(Op::Catalog { entry, dictionaries: d.dictionary, open_keys: d.open_keys });
927        Ok(())
928    }
929
930    /// `{"descriptor": ...}`: a stored function's descriptor, put and bound to nothing.
931    fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
932        let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
933        if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
934            return Err(PublishError::at(
935                n,
936                "a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
937            ));
938        }
939        let (hash, bytes) = (fd.hash(), fd.canonical());
940        self.sim.descriptors.insert(hash.clone(), bytes.clone());
941        self.ops.push(Op::Descriptor { hash, bytes, counted: true });
942        Ok(())
943    }
944
945    /// A function's descriptor (stored or formula): put, and bound when its name does not mean it.
946    fn function(
947        &mut self,
948        n: usize,
949        name: &str,
950        start: String,
951        hash: String,
952        bytes: Vec<u8>,
953    ) -> Result<(), PublishError> {
954        self.sim.descriptors.insert(hash.clone(), bytes.clone());
955        let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
956        let bind = current.as_deref() != Some(hash.as_str());
957        if bind {
958            self.check_bind(n, name, &start, false)?;
959            self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
960        }
961        self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
962        Ok(())
963    }
964
965    fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
966        let err = |d: String| PublishError::at(n, d);
967        let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
968        let start = bound_of(v, n, self.ctx.now)?;
969        let def = datumesh_formula::definition::define_class(
970            name,
971            &descriptor.to_jcs(),
972            definer(name, self.ctx.host, self.ctx.owned),
973            &self.callers(),
974            &start,
975        )
976        .map_err(|e| err(format!("define {name}: {e}")))?;
977        // put_class refuses under a suspended steward namespace, whether or not the name binds. A
978        // namespace row is a host's (9.9), so the steward namespace's host is asked: a steward with
979        // a path (datumesh.com/scripture) is not a host and Store::namespace refuses it.
980        suspended(&mut self.reads, self.store, host_of(&def.descriptor.steward.namespace), n)?;
981        let hash = def.class_hash.clone();
982        let methods: Vec<(String, String)> =
983            def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
984        let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
985        if bind {
986            self.check_bind(n, name, &start, true)?;
987            // bind_class binds every method in the same transaction (the store's rule).
988            for (mname, _) in &methods {
989                self.check_bind(n, mname, &start, false)?;
990            }
991            self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
992            for (mname, fh) in &methods {
993                self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
994            }
995        }
996        self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
997        self.ops.push(Op::Class {
998            name: name.to_string(),
999            start,
1000            hash,
1001            bytes: def.bytes,
1002            bind,
1003            methods: methods.into_iter().map(|(m, _)| m).collect(),
1004        });
1005        Ok(())
1006    }
1007
1008    /// One bind a line would make: rule A, the other kind, the namespace, and the latest binding.
1009    fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
1010        if !self.backfill && start < self.ctx.now {
1011            return Err(PublishError::typed(
1012                line,
1013                PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
1014            ));
1015        }
1016        let err = |d: String| PublishError::at(line, d);
1017        if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
1018            let kind = if class { "function" } else { "class" };
1019            return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
1020        }
1021        suspended(&mut self.reads, self.store, host_of(name), line)?;
1022        if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
1023            if start <= latest.as_str() {
1024                return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
1025            }
1026        }
1027        self.binds.push(Bind { line, name: name.into() });
1028        Ok(())
1029    }
1030
1031    /// Plans the cell lines of a chunk (CONTRACT-perf H1): each is prepared in file order up to its
1032    /// receipt; the receipts are signed on threads; the store is read once for the chunk; then each
1033    /// is checked in file order. The first failure is the chunk's, and a line that fails before
1034    /// signing is reported after the lines before it are checked.
1035    fn plan_cells(&mut self, lines: Vec<(usize, Value)>) -> Result<(), PublishError> {
1036        if lines.is_empty() {
1037            return Ok(());
1038        }
1039        let mut pending = Vec::with_capacity(lines.len());
1040        let mut failed = None;
1041        for (n, v) in &lines {
1042            match self.prepare_cell(*n, v) {
1043                Ok(p) => pending.push(p),
1044                Err(e) => {
1045                    failed = Some(e);
1046                    break;
1047                }
1048            }
1049        }
1050        let (key, host) = (self.key, self.ctx.host);
1051        let signed = parallel_owned(pending, |p| p.sign(key, host));
1052        let ids: Vec<&str> = signed.iter().flatten().map(|s| s.id.as_str()).collect();
1053        self.reads.prefetch_latest(self.store, &ids);
1054        let hashes: Vec<String> = signed
1055            .iter()
1056            .flatten()
1057            .filter(|s| !s.has_tx)
1058            .filter_map(|s| self.reads.latest_of.get(&s.id).cloned().flatten())
1059            .map(|(_, h)| h)
1060            .collect();
1061        let hashes: Vec<&str> = hashes.iter().map(String::as_str).collect();
1062        self.reads.prefetch_receipts(self.store, &hashes);
1063        for s in signed {
1064            self.check_cell(s?)?;
1065        }
1066        match failed {
1067            Some(e) => Err(e),
1068            None => Ok(()),
1069        }
1070    }
1071
1072    /// A cell line up to its receipt, as the checks of a line take them in order: the fields, the
1073    /// instants, the function as of `tx`, the descriptor, the arguments and the value (and its blob).
1074    fn prepare_cell(&mut self, n: usize, v: &Value) -> Result<Pending, PublishError> {
1075        let err = |d: String| PublishError::at(n, d);
1076        let cell_v = v.get("cell").ok_or_else(|| err("a cell without fn".into()))?;
1077        let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
1078        let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
1079        let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
1080        let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
1081        let (licence, summary) = (required("licence")?, required("summary")?);
1082        // tx and asof default to the run's clock now, each on its own (H4).
1083        let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
1084        let has_tx = field("tx").is_some();
1085        let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
1086        if tx > asof {
1087            return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
1088        }
1089        let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
1090        if asof > issued {
1091            return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
1092        }
1093        let fn_hash = function_at(&mut self.reads, self.store, &self.sim, name, &tx)
1094            .map_err(err)?
1095            .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
1096        let d = function_bytes_of(&mut self.reads, self.store, &self.sim, &fn_hash)
1097            .map_err(err)?
1098            .ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
1099        let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
1100        let mut args = Args::new();
1101        for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
1102            args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
1103        }
1104        let valid: Args = args
1105            .iter()
1106            .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
1107            .map(|(k, a)| (k.clone(), a.clone()))
1108            .collect();
1109        let source = match v.get("source") {
1110            Some(sv) => Some(source_of_line(sv).map_err(err)?),
1111            None => None,
1112        };
1113        let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
1114        let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
1115            Some(file) => {
1116                let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
1117                let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
1118                let sha = sha256_hex(&bytes);
1119                let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
1120                    .map_err(|e| err(e.to_string()))?;
1121                (scalar, Some(bytes))
1122            }
1123            None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
1124        };
1125        let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
1126        Ok(Pending {
1127            line: n,
1128            ns: ns.to_string(),
1129            cell,
1130            value,
1131            valid,
1132            tx,
1133            asof,
1134            issued,
1135            has_tx,
1136            licence,
1137            summary,
1138            blob,
1139            source,
1140        })
1141    }
1142
1143    /// The checks of a signed cell line, in file order: the skip rule of a line without `tx`, then
1144    /// the version rule. Its write is planned.
1145    fn check_cell(&mut self, s: Signed) -> Result<(), PublishError> {
1146        let line = s.line;
1147        let err = |d: String| PublishError::at(line, d);
1148        // A line without tx takes the run's clock, so a re-run at a later clock would sign the same
1149        // fact again as a new version. Re-running a file changes nothing: when the cell's latest
1150        // version is the store's and states the same thing (type, value, valid, licence, summary),
1151        // the line is skipped. A changed value is a new version, a correction.
1152        if !s.has_tx {
1153            if let Some((_, latest)) = cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1154                if let Some(old) = self.reads.receipt(self.store, &latest) {
1155                    let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
1156                    let new = json::parse(&s.receipt).map_err(|e| err(e.to_string()))?;
1157                    if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
1158                        return Ok(());
1159                    }
1160                }
1161            }
1162        }
1163        let version = match cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1164            Some((latest_tx, _)) if latest_tx.as_str() >= s.tx.as_str() => {
1165                if cell_version_asof(self.store, &self.sim, &s.id, &s.tx).map_err(err)?.as_deref()
1166                    != Some(s.hash.as_str())
1167                {
1168                    return Err(err(format!("{} has a version at or after {} that is not this receipt", s.id, s.tx)));
1169                }
1170                false
1171            }
1172            _ => {
1173                self.sim.versions.entry(s.id.clone()).or_default().push((s.tx.clone(), s.hash.clone()));
1174                true
1175            }
1176        };
1177        self.ops.push(Op::Cell { tx: s.tx, id: s.id, receipt: s.receipt, blob: s.blob, version });
1178        Ok(())
1179    }
1180}
1181
1182/// The stored descriptor of a function line's signature.
1183fn stored_value(signature: &Value) -> Value {
1184    Value::Object(vec![
1185        ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
1186        ("inputs".into(), Value::Array(vec![])),
1187        ("signature".into(), signature.clone()),
1188    ])
1189}
1190
1191/// The backfill's checks (since 0.3.0 with store 0.4.2): no name the file binds has a binding, and
1192/// no namespace of one has any binding (`has_bindings_under`).
1193fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
1194    for b in binds {
1195        let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
1196        if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
1197            return Err(refuse(format!("{} has a binding", b.name)));
1198        }
1199        let ns = namespace_of(&b.name);
1200        if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
1201            return Err(refuse(format!("{ns} has bindings")));
1202        }
1203    }
1204    Ok(())
1205}
1206
1207/// Appends a run's line to the backfill log (created with mode 0600).
1208fn append_log(log: &Path, written: &Written) -> Result<(), String> {
1209    let line = Value::Object(vec![
1210        ("at".into(), s(&SystemClock.now())),
1211        ("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
1212        ("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
1213    ])
1214    .to_jcs();
1215    let mut opts = std::fs::OpenOptions::new();
1216    opts.create(true).append(true);
1217    #[cfg(unix)]
1218    {
1219        use std::os::unix::fs::OpenOptionsExt;
1220        opts.mode(0o600);
1221    }
1222    let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
1223    let mut bytes = line;
1224    bytes.push(b'\n');
1225    f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
1226}
1227
1228/// Pass two: the writes the first pass planned, in order. Nothing here can refuse a planned
1229/// write, so a refused file has written nothing. Consecutive cell writes go in `put_cells` calls
1230/// of up to `CHUNK` on `lanes` lanes, their blobs put first (CONTRACT-perf H2, H5).
1231fn write<S: Store + ?Sized>(
1232    store: &S,
1233    ops: &[Op],
1234    cells: impl Fn(&S, &[Op], &mut Published) -> Result<(), String>,
1235) -> Result<(Published, Written), PublishError> {
1236    let mut out = Published::default();
1237    let mut written = Written::default();
1238    let err = |d: String| PublishError::at(0, d);
1239    let mut i = 0;
1240    while i < ops.len() {
1241        match &ops[i] {
1242            Op::Descriptor { hash, bytes, counted } => {
1243                let existed = store.descriptor(hash).is_some();
1244                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1245                if *counted && !existed {
1246                    out.descriptors += 1;
1247                }
1248            }
1249            Op::Function { name, start, hash, bytes, bind } => {
1250                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1251                if *bind {
1252                    store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1253                    out.functions += 1;
1254                    written.bound(name, start);
1255                }
1256            }
1257            Op::Class { name, start, hash, bytes, bind, methods } => {
1258                store.put_class(bytes).map_err(|e| err(e.to_string()))?;
1259                if *bind {
1260                    store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1261                    out.classes += 1;
1262                    written.bound(name, start);
1263                    for m in methods {
1264                        written.bound(m, start);
1265                    }
1266                }
1267            }
1268            Op::Catalog { entry, dictionaries, open_keys } => {
1269                crate::catalog::store_publisher(store, entry, dictionaries, open_keys).map_err(err)?;
1270            }
1271            Op::Cell { .. } => {
1272                let mut j = i;
1273                while j < ops.len() && j - i < CHUNK && matches!(ops[j], Op::Cell { .. }) {
1274                    j += 1;
1275                }
1276                cells(store, &ops[i..j], &mut out).map_err(err)?;
1277                i = j;
1278                continue;
1279            }
1280        }
1281        i += 1;
1282    }
1283    Ok((out, written))
1284}
1285
1286/// A run of cell writes (up to `CHUNK`) on one lane: their blobs first, then the receipts and
1287/// versions in file order in `put_cells` calls of up to `CHUNK` (CONTRACT-perf H2). No `Sync`
1288/// bound: this is what [`publish_stored_with`] runs.
1289fn write_cells_one_lane<S: Store + ?Sized>(store: &S, group: &[Op], out: &mut Published) -> Result<(), String> {
1290    let items = put_blobs_and_items(store, group, out)?;
1291    let failed = AtomicBool::new(false);
1292    let ks: Vec<usize> = (0..items.len()).collect();
1293    if let Some((_, detail)) = put_lane(store, &items, &ks, &failed) {
1294        return Err(detail);
1295    }
1296    count_cells(group, &items, out);
1297    Ok(())
1298}
1299
1300/// A run of cell writes (up to `CHUNK`): their blobs first, then their receipts and versions on
1301/// `lanes` lanes (CONTRACT-perf H5). A cell's id picks its lane (its first hex digit, mod `lanes`),
1302/// so all writes of one cell share a lane, in file order. A lane's `put_cells` calls take up to
1303/// `CHUNK` of its items. When one fails, no lane starts another call, and the error returned is
1304/// that of the earliest failing item in file order. With one lane, as [`write_cells_one_lane`].
1305fn write_cells<S: Store + Sync + ?Sized>(
1306    store: &S,
1307    group: &[Op],
1308    lanes: usize,
1309    out: &mut Published,
1310) -> Result<(), String> {
1311    if lanes <= 1 {
1312        return write_cells_one_lane(store, group, out);
1313    }
1314    let items = put_blobs_and_items(store, group, out)?;
1315    let mut by_lane: Vec<Vec<usize>> = vec![Vec::new(); lanes];
1316    for (k, op) in group.iter().enumerate() {
1317        if let Op::Cell { id, .. } = op {
1318            by_lane[lane_of(id, lanes)].push(k);
1319        }
1320    }
1321    let failed = AtomicBool::new(false);
1322    let failures: Vec<(usize, String)> = std::thread::scope(|scope| {
1323        let handles: Vec<_> = by_lane
1324            .iter()
1325            .map(|ks| {
1326                let (items, failed) = (&items, &failed);
1327                scope.spawn(move || put_lane(store, items, ks, failed))
1328            })
1329            .collect();
1330        handles.into_iter().filter_map(|h| h.join().unwrap_or_else(|p| std::panic::resume_unwind(p))).collect()
1331    });
1332    if let Some((_, detail)) = failures.into_iter().min_by_key(|(k, _)| *k) {
1333        return Err(detail);
1334    }
1335    count_cells(group, &items, out);
1336    Ok(())
1337}
1338
1339/// The blobs of a run, put in order (counted in `out`), and the run's cell writes.
1340fn put_blobs_and_items<'a, S: Store + ?Sized>(
1341    store: &S,
1342    group: &'a [Op],
1343    out: &mut Published,
1344) -> Result<Vec<CellItem<'a>>, String> {
1345    for op in group {
1346        if let Op::Cell { blob: Some(bytes), .. } = op {
1347            store.put_blob(bytes).map_err(|e| e.to_string())?;
1348            out.blobs += 1;
1349            out.blob_bytes += bytes.len() as u64;
1350        }
1351    }
1352    Ok(group
1353        .iter()
1354        .filter_map(|op| match op {
1355            Op::Cell { tx, id, receipt, version, .. } => {
1356                Some((receipt.as_slice(), if *version { Some((id.as_str(), tx.as_str())) } else { None }))
1357            }
1358            _ => None,
1359        })
1360        .collect())
1361}
1362
1363/// Counts a written run of cells in `out`.
1364fn count_cells(group: &[Op], items: &[CellItem<'_>], out: &mut Published) {
1365    out.cells += group.len();
1366    out.receipts += group.len();
1367    out.versions += items.iter().filter(|(_, v)| v.is_some()).count();
1368}
1369
1370/// The lane of a cell id: its first hex digit, mod `lanes` (CONTRACT-perf H5).
1371fn lane_of(id: &str, lanes: usize) -> usize {
1372    id.bytes().next().and_then(|b| (b as char).to_digit(16)).unwrap_or(0) as usize % lanes
1373}
1374
1375/// A cell write of a run: its receipt's bytes and its version (cell id, tx), if any.
1376type CellItem<'a> = (&'a [u8], Option<(&'a str, &'a str)>);
1377
1378/// One lane's writes: its items (indices into `items`, in file order) in `put_cells` calls of up to
1379/// `CHUNK`. Returns the index of the first item of the call that failed, and its error. No call
1380/// starts once `failed` is set.
1381fn put_lane<S: Store + ?Sized>(
1382    store: &S,
1383    items: &[CellItem<'_>],
1384    ks: &[usize],
1385    failed: &AtomicBool,
1386) -> Option<(usize, String)> {
1387    for call in ks.chunks(CHUNK) {
1388        if failed.load(Ordering::SeqCst) {
1389            return None;
1390        }
1391        let writes: Vec<CellWrite<'_>> = call.iter().map(|&k| CellWrite::new(items[k].0, items[k].1)).collect();
1392        if let Err(e) = store.put_cells(&writes) {
1393            failed.store(true, Ordering::SeqCst);
1394            return Some((call[0], e.to_string()));
1395        }
1396    }
1397    None
1398}
1399
1400/// The receipt JSON of a signed receipt: its payload's members and `sig`.
1401pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
1402    let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
1403    if let Ok(Value::Object(m)) = json::parse(r.payload()) {
1404        members.extend(m);
1405    }
1406    Value::Object(members).to_jcs()
1407}
1408
1409/// Publishes the file at `path` into `store` as the mesh `host` signing with `key` (see the
1410/// module documentation). The mesh's own namespaces are `host` alone: see [`publish_stored_in`].
1411/// The clock is the system clock, and there is no backfill. The cells are written on one lane;
1412/// see [`publish_stored_lanes`] for the parallel lanes.
1413pub fn publish_stored<S: Store + ?Sized>(
1414    store: &S,
1415    key: &SigningKey,
1416    host: &str,
1417    path: &Path,
1418) -> Result<Published, PublishError> {
1419    publish_stored_in(store, key, host, &[host.to_string()], path)
1420}
1421
1422/// As [`publish_stored`], for a mesh that owns the namespaces (hosts) `owned` (`Config::namespaces`):
1423/// a class line is accepted for a class under one of them, and refused with `E_CLASS_OWNER` (in the
1424/// [`PublishError::detail`]) under any other. Since 0.2.2. The system clock, and no backfill.
1425pub fn publish_stored_in<S: Store + ?Sized>(
1426    store: &S,
1427    key: &SigningKey,
1428    host: &str,
1429    owned: &[String],
1430    path: &Path,
1431) -> Result<Published, PublishError> {
1432    publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
1433}
1434
1435/// As [`publish_stored_in`], with the run's clock and backfill (CONTRACT-1.8b H1, rule A). The
1436/// cells are written in file order on one lane, as in 0.3.2 (CONTRACT-perf H2).
1437///
1438/// Errors: a line refused by its checks, or a binding whose start is earlier than the clock
1439/// ([`PublishErrorKind::StartBeforeNow`]), or a backfill refused ([`PublishErrorKind::Backfill`]).
1440/// Nothing is written when the first pass refuses (CONTRACT-1.7a H2).
1441pub fn publish_stored_with<S: Store + ?Sized>(
1442    store: &S,
1443    key: &SigningKey,
1444    host: &str,
1445    owned: &[String],
1446    path: &Path,
1447    opts: &PublishOptions,
1448) -> Result<Published, PublishError> {
1449    publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1450        write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1451    })
1452}
1453
1454/// As [`publish_stored_with`], with the public rule (Build 2 Part 3, D8: datumesh-http always
1455/// enforces it; `publish-stored` calls this). A file that defines a computed function (formula)
1456/// is refused unless the tests of the function (the file's, and those already held for the same
1457/// function hash) exercise every derived edge case and every branch, and all of the file's tests
1458/// pass: [`PublishErrorKind::TestsMissing`] at the function's line, [`PublishErrorKind::TestsFailing`]
1459/// at the failing test's line. Stored functions are exempt. Nothing is written when refused.
1460pub fn publish_stored_checked<S: Store + ?Sized>(
1461    store: &S,
1462    key: &SigningKey,
1463    host: &str,
1464    owned: &[String],
1465    path: &Path,
1466    opts: &PublishOptions,
1467) -> Result<Published, PublishError> {
1468    publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1469        write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1470    })
1471}
1472
1473/// As [`publish_stored_with`], with the cells written on parallel lanes (CONTRACT-perf H5, H6):
1474/// `DATUMESH_PUBLISH_LANES` (a positive integer) if set, else `min(4, available_parallelism())`.
1475/// The same receipts, versions and rows as one lane; only the order of the inserts across
1476/// different cells differs. The variable is read once per run, before the file is read.
1477pub fn publish_stored_lanes<S: Store + Sync + ?Sized>(
1478    store: &S,
1479    key: &SigningKey,
1480    host: &str,
1481    owned: &[String],
1482    path: &Path,
1483    opts: &PublishOptions,
1484) -> Result<Published, PublishError> {
1485    let lanes = publish_lanes()?;
1486    publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1487        write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1488    })
1489}
1490
1491/// The planned writes of a run as the tests see them (see [`Overlay`]).
1492fn overlay_of(ops: &[Op]) -> Result<Overlay, PublishError> {
1493    let mut overlay = Overlay::default();
1494    for op in ops {
1495        match op {
1496            Op::Function { name, start, hash, bytes, bind } => {
1497                overlay.descriptors.insert(hash.clone(), bytes.clone());
1498                if *bind {
1499                    overlay.functions.entry(name.clone()).or_default().push((start.clone(), hash.clone()));
1500                }
1501            }
1502            Op::Descriptor { hash, bytes, .. } => {
1503                overlay.descriptors.insert(hash.clone(), bytes.clone());
1504            }
1505            Op::Cell { receipt, .. } => {
1506                overlay.cell(receipt).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1507            }
1508            Op::Class { .. } | Op::Catalog { .. } => {}
1509        }
1510    }
1511    Ok(overlay)
1512}
1513
1514/// The tests of a file, run over the store and the file's planned writes (Build 2 Part 3, E5;
1515/// CONTRACT-build2-part5 F2). For a computed function the file defines (`formulas`), the public rule:
1516/// `checked` refuses the file when a test fails or the tests leave a derived case or a branch arm
1517/// unexercised. For a stored function, the tests of the file are run and stored with their results
1518/// (a cell that is not stored is `error`), and the file is never refused for them (spec 7.2: a stored
1519/// function's tests are source-conformance tests). Returns the rows of the tests to hold (the file's
1520/// and the held ones of those functions).
1521#[allow(clippy::too_many_arguments)]
1522pub(crate) fn check_tests<S: Store + ?Sized>(
1523    store: &S,
1524    overlay: &Overlay,
1525    tests: &[(usize, TestSpec)],
1526    formulas: &[(usize, String, String)],
1527    host: &str,
1528    now: &str,
1529    checked: bool,
1530) -> Result<Vec<RowWrite>, PublishError> {
1531    let mut file_tests: Vec<(usize, TestSpec, String)> = Vec::new();
1532    for (n, spec) in tests {
1533        let at = spec.asof.clone().unwrap_or_else(|| now.to_string());
1534        match testrun::binding_of(store, overlay, &spec.function, &at) {
1535            Ok(Some((hash, _))) => file_tests.push((*n, spec.clone(), hash)),
1536            Ok(None) => {
1537                return Err(PublishError::at(
1538                    *n,
1539                    format!("a test of {}: no function of that name is bound", spec.function),
1540                ))
1541            }
1542            Err(e) => return Err(PublishError::at(*n, e)),
1543        }
1544    }
1545    // The functions to run: the file's computed functions, then the stored functions of its tests.
1546    let mut groups: Vec<(usize, String, String)> = formulas.to_vec();
1547    for (n, spec, hash) in &file_tests {
1548        if !groups.iter().any(|g| &g.2 == hash) {
1549            groups.push((*n, spec.function.clone(), hash.clone()));
1550        }
1551    }
1552    let mut rows = Vec::new();
1553    for (line, name, hash) in &groups {
1554        let own: Vec<(usize, TestSpec)> =
1555            file_tests.iter().filter(|t| &t.2 == hash).map(|t| (t.0, t.1.clone())).collect();
1556        // The held tests of this function, read once (as 0.4.8 read them, before the function is built).
1557        let all_held = testrun::held_of_name(store, hash, name, now).map_err(|e| PublishError::at(*line, e))?;
1558        if !checked && own.is_empty() && all_held.is_empty() {
1559            continue;
1560        }
1561        let mut run = testrun::Run::new(store, overlay, now);
1562        let built = run.built(name, hash).map_err(|e| PublishError::at(*line, format!("{name}: {}", e.0)))?;
1563        // A computed function this file does not define keeps its behaviour: its tests are neither
1564        // checked nor stored here (as before stored functions' tests were held).
1565        if built.compiled.is_some() && !formulas.iter().any(|f| &f.2 == hash) {
1566            continue;
1567        }
1568        // A computed function's held tests are all run (its coverage is of all of them); a stored function's
1569        // only where they read a cell this file writes (CONTRACT-0.4.9 H6).
1570        let written: BTreeSet<&str> = overlay.cells.iter().filter(|c| &c.name == name).map(|c| c.id.as_str()).collect();
1571        let held: Vec<(FunctionTest, TestSpec, String)> = all_held
1572            .into_iter()
1573            .filter(|(_, _, cell)| built.compiled.is_some() || written.contains(cell.as_str()))
1574            .collect();
1575        let mut specs: Vec<(Option<usize>, TestSpec)> = own.into_iter().map(|(n, s)| (Some(n), s)).collect();
1576        for (_, spec, _) in held {
1577            if !specs.iter().any(|(_, s)| s.case == spec.case) {
1578                specs.push((None, spec));
1579            }
1580        }
1581        let ran: Vec<Ran> = specs.iter().map(|(_, s)| run_test(store, overlay, now, s)).collect();
1582        let Some(compiled) = built.compiled.as_ref() else {
1583            // A stored function: its tests are stored and counted; it has no branch arms.
1584            for ((own_n, spec), r) in specs.iter().zip(&ran) {
1585                let (result, detail) = testrun::result_of(spec, r);
1586                rows.push(RowWrite {
1587                    fn_hash: hash.clone(),
1588                    case: spec.case.clone(),
1589                    jcs: spec.jcs.clone(),
1590                    signed_by: host.to_string(),
1591                    result,
1592                    detail,
1593                    branches_total: 0,
1594                    branches_covered: 0,
1595                    fn_name: row_name(*own_n, spec, name),
1596                });
1597            }
1598            continue;
1599        };
1600        if checked {
1601            let mut failing: Vec<(usize, String)> = Vec::new();
1602            for ((n, spec), r) in specs.iter().zip(&ran) {
1603                if let Some(n) = n {
1604                    let (result, detail) = testrun::result_of(spec, r);
1605                    if result != "pass" {
1606                        failing.push((*n, format!("{name}: test {} {detail}", spec.case)));
1607                    }
1608                }
1609            }
1610            failing.sort_by_key(|f| f.0);
1611            if let Some((n, detail)) = failing.into_iter().next() {
1612                return Err(PublishError::typed(n, PublishErrorKind::TestsFailing(detail)));
1613            }
1614            let refs: Vec<&Ran> = ran.iter().collect();
1615            let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1616            if !cov.uncovered.is_empty() || !cov.unexercised.is_empty() {
1617                let mut text = format!("{name} is missing tests:");
1618                if !cov.uncovered.is_empty() {
1619                    let cases: Vec<&str> = cov.uncovered.iter().map(|c| c.0.as_str()).collect();
1620                    text += &format!(" uncovered cases {};", cases.join(", "));
1621                }
1622                if !cov.unexercised.is_empty() {
1623                    text += &format!(" unexercised branches {}", cov.unexercised.join(", "));
1624                }
1625                return Err(PublishError::typed(
1626                    *line,
1627                    PublishErrorKind::TestsMissing(text.trim_end_matches(';').to_string()),
1628                ));
1629            }
1630        }
1631        let refs: Vec<&Ran> = ran.iter().collect();
1632        let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1633        for ((own_n, spec), r) in specs.iter().zip(&ran) {
1634            let (result, detail) = testrun::result_of(spec, r);
1635            rows.push(RowWrite {
1636                fn_hash: hash.clone(),
1637                case: spec.case.clone(),
1638                jcs: spec.jcs.clone(),
1639                signed_by: host.to_string(),
1640                result,
1641                detail,
1642                branches_total: cov.branches,
1643                branches_covered: cov.covered,
1644                fn_name: row_name(*own_n, spec, name),
1645            });
1646        }
1647    }
1648    Ok(rows)
1649}
1650
1651/// As [`publish_stored_lanes`], with the public rule (see [`publish_stored_checked`]): the path of
1652/// `datumesh-mesh publish-stored` (D8).
1653pub fn publish_stored_checked_lanes<S: Store + Sync + ?Sized>(
1654    store: &S,
1655    key: &SigningKey,
1656    host: &str,
1657    owned: &[String],
1658    path: &Path,
1659    opts: &PublishOptions,
1660) -> Result<Published, PublishError> {
1661    let lanes = publish_lanes()?;
1662    publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1663        write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1664    })
1665}
1666
1667/// The first pass (the planner) and the second pass (`write_ops`) of a publish.
1668#[allow(clippy::too_many_arguments)]
1669fn publish_planned<S: Store + ?Sized>(
1670    store: &S,
1671    key: &SigningKey,
1672    host: &str,
1673    owned: &[String],
1674    path: &Path,
1675    opts: &PublishOptions,
1676    checked: bool,
1677    write_ops: impl FnOnce(&S, &[Op]) -> Result<(Published, Written), PublishError>,
1678) -> Result<Published, PublishError> {
1679    let lines = read_lines(path)?;
1680    // The key's start: the earliest `issued`.
1681    let key_from = store
1682        .namespace(host)
1683        .ok()
1684        .flatten()
1685        .and_then(|n| json::parse(&n.keydoc).ok())
1686        .and_then(|d| match d.get("keys") {
1687            Some(Value::Array(ks)) => ks
1688                .iter()
1689                .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
1690                .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
1691            _ => None,
1692        })
1693        .ok_or_else(|| {
1694            PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
1695        })?;
1696    let now = opts.clock.now();
1697    let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
1698    let mut planner = Planner {
1699        store,
1700        key,
1701        ctx,
1702        backfill: opts.backfill.is_some(),
1703        sim: Simulation::default(),
1704        reads: Reads::default(),
1705        ops: Vec::new(),
1706        binds: Vec::new(),
1707        tests: Vec::new(),
1708        formulas: Vec::new(),
1709    };
1710    planner.run(&lines)?;
1711    let Planner { ops, binds, tests, formulas, .. } = planner;
1712    if opts.backfill.is_some() {
1713        backfill_check(store, &binds)?;
1714    }
1715    // The tests of the file's computed functions, run over the store and the file's own writes:
1716    // refused before anything is written (Build 2 Part 3, D8).
1717    let overlay = overlay_of(&ops)?;
1718    let rows = check_tests(store, &overlay, &tests, &formulas, host, &now, checked)?;
1719    let (out, written) = write_ops(store, &ops)?;
1720    index_cell_values(store, &ops).map_err(|e| PublishError::at(0, format!("catalogue: {e}")))?;
1721    testrun::apply_rows(store, &rows, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1722    let rebound: Vec<String> = ops
1723        .iter()
1724        .filter_map(|op| match op {
1725            Op::Function { name, bind: true, .. } => Some(name.clone()),
1726            _ => None,
1727        })
1728        .collect();
1729    testrun::rerun_readers(store, &rebound, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1730    // The stored functions' tests read the cells this file writes, and the names it rebinds.
1731    let rebound: BTreeSet<String> = rebound.into_iter().collect();
1732    let written_cells: Vec<(String, String)> = overlay.cells.iter().map(|c| (c.name.clone(), c.id.clone())).collect();
1733    testrun::rerun_stored_tests(store, &rebound, &written_cells, &now)
1734        .map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1735    if let Some(log) = &opts.backfill {
1736        if out.functions + out.classes + out.versions > 0 {
1737            append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
1738        }
1739    }
1740    Ok(out)
1741}