Skip to main content

datumesh_http/
publish.rs

1//! Publishing stored cells as the mesh (`datumesh-mesh publish-stored`): a JSON lines file of
2//! stored functions and their cells, signed with the mesh's key, put in the store.
3//!
4//! Each line is one of:
5//!
6//! * a descriptor (since 0.3.1, CONTRACT-1.7a H1): `{"descriptor": <function descriptor>}`: the
7//!   descriptor is put with `put_descriptor` and binds nothing. Only a stored function is
8//!   accepted (`code.lang` = `stored`, `code.body` = `""`, `inputs` = `[]`); any other is refused
9//!   at its line. A class later in the file (or in a later run) may name its hash as a method;
10//! * a function: `{"function": "<ns>/<name>", "signature": {...}, "bound": "<instant>"}`: the
11//!   descriptor `{"code": {"body": "", "lang": "stored"}, "inputs": [], "signature": ...}`
12//!   is put and the name bound to it from `bound` (unless it already is);
13//! * a formula function: `{"formula": "<ns>/<name>", "source": "<formula>", "signature":
14//!   {...}, "bound": "<instant>"}`: defined with `datumesh_formula::definition::define_function`
15//!   (its callees as bound at `bound`) and published (`put_descriptor`, `bind_name` from
16//!   `bound`) unless the name is already bound to that function; its cells are evaluated on
17//!   demand by the mesh;
18//! * a class (since 0.2.2, spec-0.5.0, 16.2): `{"class": "<ns>/<name>", "descriptor": {...as
19//!   written...}, "bound": "<instant>"}`: defined with the formula crate's `define_class` (core's
20//!   rules: the class is under one of the mesh's own namespaces, else `E_CLASS_OWNER`; the method
21//!   functions are those bound as of `bound`), `put_class`, and `bind_class` from `bound` unless the
22//!   name is already bound to that class hash. Method functions come first in the file (or in an
23//!   earlier one), `bound` later than theirs. A line with both `class` and `descriptor` is a class
24//!   line;
25//! * a cell: `{"cell": {"fn": "<ns>/<name>", "args": {<name>: <typed scalar>}}, "value": <typed
26//!   scalar> or {"blob": "<file>", "media_type": "<type>"}, "tx"?, "asof"?, "issued"?, "licence",
27//!   "summary"}`: a blob's file (relative to the JSON lines file) is put with `put_blob` and the
28//!   value is `<sha256>:<length>:<media type>`; the receipt is signed with the mesh's key
29//!   (`computed_by` the mesh host, `publisher` the cell's namespace, `valid` its valid-time
30//!   arguments; `issued` defaults to the later of `asof` and the key's start) and stored with
31//!   `put_receipt` and `put_version`. `tx` and `asof` default to the run's clock now (since
32//!   0.3.1, H4), each independently.
33//!
34//! The function of a cell is the one its name is bound to as of the cell's `tx`. Re-running a
35//! file changes nothing (a cell line without `tx` whose latest version states the same thing is
36//! skipped, since 0.3.1): descriptors and blobs are content-addressed, signing is deterministic
37//! (the same receipt, the same hash), and a version already there is skipped. The key is never
38//! logged.
39//!
40//! Rule A (since 0.3.0, CONTRACT-1.8b H1): no binding starts earlier than the mesh clock. The
41//! start of a binding (a name's, a class's, or a class method's `N/m`) is its `bound`, or the
42//! clock's now when `bound` is left out, and a start earlier than now is refused with
43//! [`PublishErrorKind::StartBeforeNow`]. A `--backfill` run ([`PublishOptions::backfill`]) lifts
44//! the start check, and is refused ([`PublishErrorKind::Backfill`]) when a name it would bind has
45//! a binding, or when a namespace of such a name has any binding (store 0.4.2's
46//! `has_bindings_under`). A run that writes appends one line to the log: `{"at": …, "namespaces":
47//! […], "starts": […]}`, a JCS object.
48//!
49//! No partial writes (since 0.3.1, CONTRACT-1.7a H2): the file is read in two passes. The first
50//! plans every write the second one makes, and checks each as the store will: a bind is refused
51//! when its start is not later than the latest binding of its name (the store's, or one planned
52//! earlier in the file, W1b), when the name is of the other kind (a class or a function), or when
53//! its namespace is suspended; a cell's function must be bound as of its `tx` (counting the
54//! earlier binds of the file), its blob must be readable, `tx <= asof <= issued`, and a version
55//! at or after its `tx` must be this receipt. Receipts are signed in the first pass (signing is
56//! deterministic), so their hashes are known. A file that the first pass refuses writes nothing.
57
58use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
59use std::io::{BufRead, Write};
60use std::path::{Path, PathBuf};
61use std::sync::atomic::{AtomicBool, Ordering};
62use std::sync::Arc;
63
64use datumesh_core::json::{self, Value};
65use datumesh_core::{
66    sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey, Type,
67};
68use datumesh_engine::{Clock, SystemClock};
69use datumesh_formula::definition::{Callee, Callees, ClassCallee};
70use datumesh_store::{Binding, CellWrite, Namespace, Store};
71
72use crate::config::ConfigError;
73
74/// The end of time: a binding as of it is the latest one.
75const END: &str = "9999-12-31T23:59:59.999999Z";
76
77/// Cell lines planned together, and cell writes per `put_cells` call (CONTRACT-perf H1, H2).
78const CHUNK: usize = 1_000;
79
80/// The environment variable of the publish lanes (CONTRACT-perf H5), read once per run.
81const LANES_VAR: &str = "DATUMESH_PUBLISH_LANES";
82
83/// The lanes of a run (CONTRACT-perf H5): `DATUMESH_PUBLISH_LANES` (a positive integer), else
84/// `min(4, available_parallelism)`, at least 1. An invalid value refuses the run before any write.
85fn publish_lanes() -> Result<usize, PublishError> {
86    let bad = |detail: String| PublishError::at(0, ConfigError::new(LANES_VAR, detail).to_string());
87    match std::env::var(LANES_VAR) {
88        Err(std::env::VarError::NotPresent) => Ok(std::thread::available_parallelism().map_or(1, |n| n.get()).min(4)),
89        Ok(v) => match v.parse::<usize>() {
90            Ok(n) if n > 0 => Ok(n),
91            _ => Err(bad(format!("must be a positive integer, not {v:?}"))),
92        },
93        Err(e) => Err(bad(e.to_string())),
94    }
95}
96
97/// What a run put.
98#[non_exhaustive]
99#[derive(Clone, Debug, Default, PartialEq, Eq)]
100pub struct Published {
101    /// Functions newly bound (stored and formula).
102    pub functions: usize,
103    /// Classes newly bound (since 0.2.2).
104    pub classes: usize,
105    /// Cell lines read.
106    pub cells: usize,
107    /// Receipts stored (new or already there).
108    pub receipts: usize,
109    /// Versions newly added.
110    pub versions: usize,
111    /// Blobs put (new or already there).
112    pub blobs: usize,
113    /// Blob bytes put.
114    pub blob_bytes: u64,
115    /// Descriptors newly put by descriptor lines (since 0.3.1); a re-run counts 0.
116    pub descriptors: u64,
117}
118
119/// How a run reads the clock and whether it backfills (CONTRACT-1.8b H1).
120#[derive(Clone)]
121pub struct PublishOptions {
122    /// The mesh clock: the start of a binding left out is its now, and a start earlier than it is
123    /// refused (rule A).
124    pub clock: Arc<dyn Clock + Send + Sync>,
125    /// The operator log of a `--backfill` run (`DATUMESH_BACKFILL_LOG`). `Some` lifts the start
126    /// check, for a file whose names and namespaces have no bindings yet, and appends a line to
127    /// the log for each run that writes.
128    pub backfill: Option<PathBuf>,
129}
130
131impl PublishOptions {
132    /// The system clock, and no backfill.
133    pub fn system() -> PublishOptions {
134        PublishOptions { clock: Arc::new(SystemClock), backfill: None }
135    }
136}
137
138/// Why a line (or the file, at line 0) was refused.
139#[non_exhaustive]
140#[derive(Clone, Debug, PartialEq, Eq)]
141pub enum PublishErrorKind {
142    /// Any other refusal: [`PublishError::detail`] says what.
143    Line,
144    /// A binding's start is earlier than the mesh clock (rule A).
145    StartBeforeNow {
146        /// The name (or class name, or method name) that would bind.
147        name: String,
148        /// Its start.
149        start: String,
150        /// The mesh clock.
151        now: String,
152    },
153    /// A `--backfill` run that would write over a binding, or under a namespace with one.
154    Backfill(String),
155    /// `import-vectors`: a name the vectors would bind is under a namespace the mesh owns.
156    OwnedVectorName(String),
157}
158
159impl std::fmt::Display for PublishErrorKind {
160    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
161        match self {
162            PublishErrorKind::Line => f.write_str("refused"),
163            PublishErrorKind::StartBeforeNow { name, start, now } => {
164                write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
165            }
166            PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
167            PublishErrorKind::OwnedVectorName(name) => {
168                write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
169            }
170        }
171    }
172}
173
174/// Why a run stopped: the line, what was wrong, and its typed kind.
175#[derive(Clone, Debug, PartialEq, Eq)]
176pub struct PublishError {
177    /// The line number (1-based; 0 for the file).
178    pub line: usize,
179    /// Why, in words.
180    pub detail: String,
181    /// Why, typed.
182    pub kind: PublishErrorKind,
183}
184
185impl PublishError {
186    fn at(line: usize, detail: impl Into<String>) -> PublishError {
187        PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
188    }
189
190    pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
191        PublishError { line, detail: kind.to_string(), kind }
192    }
193}
194
195impl std::fmt::Display for PublishError {
196    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
197        write!(f, "line {}: {}", self.line, self.detail)
198    }
199}
200
201impl std::error::Error for PublishError {}
202
203fn s(v: &str) -> Value {
204    Value::String(v.to_string())
205}
206
207/// The namespace of a name: the name minus its last `/` segment.
208fn namespace_of(name: &str) -> &str {
209    name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
210}
211
212/// The host of a name: its first segment.
213fn host_of(name: &str) -> &str {
214    name.split('/').next().unwrap_or("")
215}
216
217/// The definer of a class `name`: its own namespace when the mesh owns the name's host, else the
218/// mesh's host (which no class name under another host has, so core answers `E_CLASS_OWNER`).
219fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
220    let host_of_name = name.split('/').next().unwrap_or("");
221    if owned.iter().any(|o| o == host_of_name) {
222        namespace_of(name)
223    } else {
224        host
225    }
226}
227
228/// The start of a line's binding: its `bound`, or the clock's now when it is left out.
229fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
230    match v.get("bound") {
231        None => Ok(now.to_string()),
232        Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
233    }
234}
235
236/// The bindings and versions a pass one has planned so far: the file's own, which are seen before
237/// the store's.
238#[derive(Default)]
239struct Simulation {
240    /// Function name to the (tx, function hash) of each bind the file would make.
241    functions: BTreeMap<String, Vec<(String, String)>>,
242    /// Class name to the (tx, class hash) of each class bind the file would make.
243    classes: BTreeMap<String, Vec<(String, String)>>,
244    /// Canonical bytes by function hash, of the descriptors the file would store.
245    descriptors: BTreeMap<String, Vec<u8>>,
246    /// Canonical bytes by class hash, of the classes the file would store.
247    class_bytes: BTreeMap<String, Vec<u8>>,
248    /// Cell id to the (tx, receipt hash) of each version the file would add, in tx order.
249    versions: BTreeMap<String, Vec<(String, String)>>,
250}
251
252impl Simulation {
253    /// The latest of `binds`' entries for `name` with tx <= `at`.
254    fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
255        binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
256    }
257}
258
259/// The callees of a definition: the store's bindings, and with `sim` the file's own over them.
260struct Callers<'a, S: Store + ?Sized> {
261    store: &'a S,
262    sim: Option<&'a Simulation>,
263}
264
265impl<S: Store + ?Sized> Callers<'_, S> {
266    fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
267        if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
268            return Ok(Some(b.clone()));
269        }
270        Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
271    }
272
273    fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
274        if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
275            return Ok(Some(b.clone()));
276        }
277        self.store.class(hash).map_err(|e| e.to_string())
278    }
279}
280
281impl<S: Store + ?Sized> Callees for Callers<'_, S> {
282    fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
283        let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
284        let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
285        let (fn_hash, start) = match (stored, simmed) {
286            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
287            (Some(b), _) => (b.fn_hash, b.tx),
288            (None, Some((t, h))) => (h, t),
289            (None, None) => return Ok(None),
290        };
291        let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
292        let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
293        Ok(Some(Callee { fn_hash, start, descriptor }))
294    }
295
296    fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
297        let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
298        let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
299        let (class_hash, start) = match (stored, simmed) {
300            (Some(b), Some((t, h))) if t >= b.tx => (h, t),
301            (Some(b), _) => (b.class_hash, b.tx),
302            (None, Some((t, h))) => (h, t),
303            (None, None) => return Ok(None),
304        };
305        let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
306        let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
307        Ok(Some(ClassCallee { class_hash, start, descriptor }))
308    }
309
310    fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
311        let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
312        ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
313    }
314
315    fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
316        let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
317        json::parse(&bytes).map(Some).map_err(|e| e.to_string())
318    }
319}
320
321/// What a run reads from the store again and again, kept for the run (CONTRACT-perf H1). The
322/// store does not change during the first pass, so each answer is the same whenever it is asked,
323/// and a failed read is kept as its message, which is what the check that asks it reports. A
324/// prefetch that fails keeps nothing: each line then asks for itself and fails at its own line.
325#[derive(Default)]
326struct Reads {
327    bindings: HashMap<(String, String), Result<Option<Binding>, String>>,
328    descriptors: HashMap<String, Option<Vec<u8>>>,
329    namespaces: HashMap<String, Result<Option<Namespace>, String>>,
330    /// The store's latest version of a cell (tx, receipt hash), prefetched per chunk.
331    latest_of: HashMap<String, Option<(String, String)>>,
332    /// The store's receipt, prefetched for the latest receipts a skip check reads.
333    receipt_of: HashMap<String, Option<Vec<u8>>>,
334}
335
336impl Reads {
337    fn binding<S: Store + ?Sized>(&mut self, store: &S, name: &str, at: &str) -> Result<Option<Binding>, String> {
338        let key = (name.to_string(), at.to_string());
339        if let Some(r) = self.bindings.get(&key) {
340            return r.clone();
341        }
342        let r = store.resolve_binding(name, at).map_err(|e| e.to_string());
343        self.bindings.insert(key, r.clone());
344        r
345    }
346
347    fn descriptor<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
348        if let Some(b) = self.descriptors.get(hash) {
349            return b.clone();
350        }
351        let b = store.descriptor(hash).map(|b| b.into_owned());
352        self.descriptors.insert(hash.to_string(), b.clone());
353        b
354    }
355
356    fn namespace<S: Store + ?Sized>(&mut self, store: &S, host: &str) -> Result<Option<Namespace>, String> {
357        if let Some(r) = self.namespaces.get(host) {
358            return r.clone();
359        }
360        let r = store.namespace(host).map_err(|e| e.to_string());
361        self.namespaces.insert(host.to_string(), r.clone());
362        r
363    }
364
365    /// The store's latest version of `id`: prefetched, else asked now.
366    fn latest<S: Store + ?Sized>(&mut self, store: &S, id: &str) -> Result<Option<(String, String)>, String> {
367        if let Some(l) = self.latest_of.get(id) {
368            return Ok(l.clone());
369        }
370        store.latest(id).map(|v| v.map(|v| (v.tx, v.receipt_hash))).map_err(|e| e.to_string())
371    }
372
373    /// The stored receipt `hash`, as `receipt` answers: prefetched, else asked now.
374    fn receipt<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
375        if let Some(b) = self.receipt_of.get(hash) {
376            return b.clone();
377        }
378        store.receipt(hash).map(|b| b.into_owned())
379    }
380
381    /// One `latest_many` for the ids not read yet.
382    fn prefetch_latest<S: Store + ?Sized>(&mut self, store: &S, ids: &[&str]) {
383        let mut seen = HashSet::new();
384        let missing: Vec<&str> =
385            ids.iter().copied().filter(|id| !self.latest_of.contains_key(*id) && seen.insert(*id)).collect();
386        if missing.is_empty() {
387            return;
388        }
389        if let Ok(found) = store.latest_many(&missing) {
390            for (id, v) in missing.iter().zip(found) {
391                self.latest_of.insert(id.to_string(), v.map(|v| (v.tx, v.receipt_hash)));
392            }
393        }
394    }
395
396    /// One `receipts_many` for the hashes not read yet.
397    fn prefetch_receipts<S: Store + ?Sized>(&mut self, store: &S, hashes: &[&str]) {
398        let mut seen = HashSet::new();
399        let missing: Vec<&str> =
400            hashes.iter().copied().filter(|h| !self.receipt_of.contains_key(*h) && seen.insert(*h)).collect();
401        if missing.is_empty() {
402            return;
403        }
404        if let Ok(found) = store.receipts_many(&missing) {
405            for (h, b) in missing.iter().zip(found) {
406                self.receipt_of.insert(h.to_string(), b);
407            }
408        }
409    }
410}
411
412/// `f` over `items` on up to `available_parallelism()` threads; the results in item order.
413fn parallel_owned<T: Send, R: Send>(items: Vec<T>, f: impl Fn(T) -> R + Sync) -> Vec<R> {
414    let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(items.len());
415    if threads <= 1 {
416        return items.into_iter().map(f).collect();
417    }
418    let size = items.len().div_ceil(threads);
419    let mut parts: Vec<Vec<T>> = Vec::with_capacity(threads);
420    let mut rest = items.into_iter();
421    loop {
422        let part: Vec<T> = rest.by_ref().take(size).collect();
423        if part.is_empty() {
424            break;
425        }
426        parts.push(part);
427    }
428    std::thread::scope(|s| {
429        let handles: Vec<_> = parts
430            .into_iter()
431            .map(|part| {
432                let f = &f;
433                s.spawn(move || part.into_iter().map(f).collect::<Vec<R>>())
434            })
435            .collect();
436        handles.into_iter().flat_map(|h| h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect()
437    })
438}
439
440/// The function hash `name` means at the end of time: the file's own over the store's.
441fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
442    if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
443        return Ok(Some(h));
444    }
445    store.resolve_name(name, END).map_err(|e| e.to_string())
446}
447
448/// The class hash `name` means at the end of time: the file's own over the store's.
449fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
450    if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
451        return Ok(Some(h));
452    }
453    store.resolve_class(name, END).map_err(|e| e.to_string())
454}
455
456/// The function hash `name` means as of `at`, counting the file's earlier binds.
457fn function_at<S: Store + ?Sized>(
458    reads: &mut Reads,
459    store: &S,
460    sim: &Simulation,
461    name: &str,
462    at: &str,
463) -> Result<Option<String>, String> {
464    let stored = reads.binding(store, name, at)?;
465    let simmed = Simulation::latest(&sim.functions, name, at);
466    Ok(match (stored, simmed) {
467        (Some(b), Some((t, h))) if t >= b.tx => Some(h),
468        (Some(b), _) => Some(b.fn_hash),
469        (None, Some((_, h))) => Some(h),
470        (None, None) => None,
471    })
472}
473
474/// The latest binding tx of `name` (a class's, or a function's), the store's or the file's.
475fn latest_bind<S: Store + ?Sized>(
476    store: &S,
477    sim: &Simulation,
478    name: &str,
479    class: bool,
480) -> Result<Option<String>, String> {
481    let stored: Option<String> = if class {
482        store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
483    } else {
484        store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
485    };
486    let binds = if class { &sim.classes } else { &sim.functions };
487    let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
488    Ok(match (stored, planned) {
489        (Some(a), Some(b)) => Some(a.max(b)),
490        (a, b) => a.or(b),
491    })
492}
493
494/// Whether `name` is bound as the other kind: a function for a class, a class for a function.
495fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
496    if class {
497        Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
498    } else {
499        Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
500            || sim.classes.contains_key(name))
501    }
502}
503
504/// The latest version of a cell, the store's or the file's: its tx and receipt hash.
505fn cell_latest<S: Store + ?Sized>(
506    reads: &mut Reads,
507    store: &S,
508    sim: &Simulation,
509    id: &str,
510) -> Result<Option<(String, String)>, String> {
511    let stored = reads.latest(store, id)?;
512    let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
513    Ok(match (stored, planned) {
514        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
515        (a, b) => a.or(b),
516    })
517}
518
519/// The receipt hash of a cell's version as of `at`, the store's or the file's.
520fn cell_version_asof<S: Store + ?Sized>(
521    store: &S,
522    sim: &Simulation,
523    id: &str,
524    at: &str,
525) -> Result<Option<String>, String> {
526    let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
527    let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
528    let latest = match (stored, planned) {
529        (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
530        (a, b) => a.or(b),
531    };
532    Ok(latest.map(|x| x.1))
533}
534
535/// The descriptor bytes of a function hash: the file's own, else the store's (memoized).
536fn function_bytes_of<S: Store + ?Sized>(
537    reads: &mut Reads,
538    store: &S,
539    sim: &Simulation,
540    hash: &str,
541) -> Result<Option<Vec<u8>>, String> {
542    if let Some(b) = sim.descriptors.get(hash) {
543        return Ok(Some(b.clone()));
544    }
545    Ok(reads.descriptor(store, hash))
546}
547
548/// Whether a namespace is suspended (the store refuses binds and class puts under it).
549fn suspended<S: Store + ?Sized>(reads: &mut Reads, store: &S, host: &str, line: usize) -> Result<(), PublishError> {
550    match reads.namespace(store, host) {
551        Ok(Some(n)) if n.suspended_at.is_some() => {
552            Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
553        }
554        Ok(_) => Ok(()),
555        Err(e) => Err(PublishError::at(line, e)),
556    }
557}
558
559/// A binding a run would make: the line that makes it and its name.
560struct Bind {
561    line: usize,
562    name: String,
563}
564
565/// What a run writes, for the backfill log: the namespaces and starts of its binds.
566#[derive(Default)]
567struct Written {
568    namespaces: BTreeSet<String>,
569    starts: BTreeSet<String>,
570}
571
572impl Written {
573    fn bound(&mut self, name: &str, start: &str) {
574        self.namespaces.insert(namespace_of(name).to_string());
575        self.starts.insert(start.to_string());
576    }
577}
578
579/// The per-run values both passes use.
580struct Ctx<'a> {
581    host: &'a str,
582    owned: &'a [String],
583    /// The start of the mesh's key.
584    key_from: &'a str,
585    /// The clock's now, read once for the run.
586    now: &'a str,
587    /// The directory of the file (blobs are relative to it).
588    base: &'a Path,
589}
590
591/// One write the second pass makes, planned (and checked) by the first.
592enum Op {
593    /// A descriptor line: put; counted when newly put.
594    Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
595    /// A function (stored or formula): put its descriptor, and bind the name when `bind`.
596    Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
597    /// A class: put it, and bind it (its methods with it, in the store) when `bind`.
598    Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
599    /// A cell: its blob, its receipt, and the version when `version`.
600    Cell { tx: String, id: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
601}
602
603/// The lines of a file, numbered, without the blank ones.
604fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
605    let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
606    let mut out = Vec::new();
607    for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
608        let n = i + 1;
609        let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
610        if !line.trim().is_empty() {
611            out.push((n, line));
612        }
613    }
614    Ok(out)
615}
616
617/// Whether a line is a cell line: none of the other kinds (the dispatch order of `plan_line`).
618fn is_cell_line(v: &Value) -> bool {
619    v.get("class").and_then(Value::as_str).is_none()
620        && v.get("descriptor").is_none()
621        && v.get("function").and_then(Value::as_str).is_none()
622        && v.get("formula").and_then(Value::as_str).is_none()
623        && v.get("cell").is_some()
624}
625
626/// A cell line prepared for signing: everything its receipt takes, checked up to there.
627struct Pending {
628    line: usize,
629    ns: String,
630    cell: Cell,
631    value: Scalar,
632    valid: Args,
633    tx: String,
634    asof: String,
635    issued: String,
636    has_tx: bool,
637    licence: String,
638    summary: String,
639    blob: Option<Vec<u8>>,
640}
641
642impl Pending {
643    /// Signs the receipt (deterministic) and takes its hash and cell id. Runs on any thread.
644    fn sign(self, key: &SigningKey, host: &str) -> Result<Signed, PublishError> {
645        let line = self.line;
646        let err = move |d: String| PublishError::at(line, d);
647        let r = ReceiptBuilder::new()
648            .cell(self.cell.clone())
649            .value(self.value)
650            .valid(self.valid)
651            .asof(self.asof.as_str())
652            .tx(self.tx.as_str())
653            .issued(self.issued.as_str())
654            .publisher(self.ns.as_str())
655            .signer_host(host)
656            .licence(self.licence.as_str())
657            .summary(self.summary.as_str())
658            .sign(key)
659            .map_err(|e| err(e.to_string()))?;
660        let receipt = receipt_json(&r);
661        let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
662        let id = self.cell.id();
663        Ok(Signed { line, has_tx: self.has_tx, tx: self.tx, id, hash, receipt, blob: self.blob })
664    }
665}
666
667/// A cell line signed: what its checks and its write take.
668struct Signed {
669    line: usize,
670    has_tx: bool,
671    tx: String,
672    id: String,
673    hash: String,
674    receipt: Vec<u8>,
675    blob: Option<Vec<u8>>,
676}
677
678/// Pass one: plans the writes of the lines and checks each as the store will (CONTRACT-1.7a H2).
679struct Planner<'a, S: Store + ?Sized> {
680    store: &'a S,
681    key: &'a SigningKey,
682    ctx: Ctx<'a>,
683    backfill: bool,
684    sim: Simulation,
685    reads: Reads,
686    ops: Vec<Op>,
687    binds: Vec<Bind>,
688}
689
690impl<S: Store + ?Sized> Planner<'_, S> {
691    fn callers(&self) -> Callers<'_, S> {
692        Callers { store: self.store, sim: Some(&self.sim) }
693    }
694
695    /// Plans the lines in file order. Cell lines are planned in chunks of up to `CHUNK`
696    /// consecutive ones; any other line ends a chunk (CONTRACT-perf H1).
697    fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
698        let mut cells: Vec<(usize, Value)> = Vec::new();
699        for (n, text) in lines {
700            let n = *n;
701            match json::parse(text.as_bytes()).map_err(|e| PublishError::at(n, e.to_string())) {
702                Ok(v) if is_cell_line(&v) => {
703                    cells.push((n, v));
704                    if cells.len() == CHUNK {
705                        self.plan_cells(std::mem::take(&mut cells))?;
706                    }
707                }
708                other => {
709                    // The cells before this line are checked first: a failure of either is the
710                    // file's first.
711                    self.plan_cells(std::mem::take(&mut cells))?;
712                    self.plan_line(n, &other?)?;
713                }
714            }
715        }
716        self.plan_cells(cells)
717    }
718
719    /// One line that is not a cell (the dispatch of the lines before cells, as it was).
720    fn plan_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
721        let err = |d: String| PublishError::at(n, d);
722        if let Some(name) = v.get("class").and_then(Value::as_str) {
723            self.class_line(n, name, v)?;
724        } else if let Some(d) = v.get("descriptor") {
725            self.descriptor_line(n, d)?;
726        } else if let Some(name) = v.get("function").and_then(Value::as_str) {
727            let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
728            let start = bound_of(v, n, self.ctx.now)?;
729            let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
730            self.function(n, name, start, fd.hash(), fd.canonical())?;
731        } else if let Some(name) = v.get("formula").and_then(Value::as_str) {
732            let source =
733                v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
734            let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
735            let start = bound_of(v, n, self.ctx.now)?;
736            let def = datumesh_formula::definition::define_function(
737                name,
738                source,
739                sig,
740                &Default::default(),
741                &self.callers(),
742                &start,
743            )
744            .map_err(|e| err(format!("define {name}: {e}")))?;
745            self.function(n, name, start, def.fn_hash, def.bytes)?;
746        } else {
747            return Err(err("neither a function nor a cell".into()));
748        }
749        Ok(())
750    }
751
752    /// `{"descriptor": ...}`: a stored function's descriptor, put and bound to nothing.
753    fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
754        let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
755        if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
756            return Err(PublishError::at(
757                n,
758                "a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
759            ));
760        }
761        let (hash, bytes) = (fd.hash(), fd.canonical());
762        self.sim.descriptors.insert(hash.clone(), bytes.clone());
763        self.ops.push(Op::Descriptor { hash, bytes, counted: true });
764        Ok(())
765    }
766
767    /// A function's descriptor (stored or formula): put, and bound when its name does not mean it.
768    fn function(
769        &mut self,
770        n: usize,
771        name: &str,
772        start: String,
773        hash: String,
774        bytes: Vec<u8>,
775    ) -> Result<(), PublishError> {
776        self.sim.descriptors.insert(hash.clone(), bytes.clone());
777        let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
778        let bind = current.as_deref() != Some(hash.as_str());
779        if bind {
780            self.check_bind(n, name, &start, false)?;
781            self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
782        }
783        self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
784        Ok(())
785    }
786
787    fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
788        let err = |d: String| PublishError::at(n, d);
789        let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
790        let start = bound_of(v, n, self.ctx.now)?;
791        let def = datumesh_formula::definition::define_class(
792            name,
793            &descriptor.to_jcs(),
794            definer(name, self.ctx.host, self.ctx.owned),
795            &self.callers(),
796            &start,
797        )
798        .map_err(|e| err(format!("define {name}: {e}")))?;
799        // put_class refuses under a suspended steward namespace, whether or not the name binds. A
800        // namespace row is a host's (9.9), so the steward namespace's host is asked: a steward with
801        // a path (datumesh.com/scripture) is not a host and Store::namespace refuses it.
802        suspended(&mut self.reads, self.store, host_of(&def.descriptor.steward.namespace), n)?;
803        let hash = def.class_hash.clone();
804        let methods: Vec<(String, String)> =
805            def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
806        let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
807        if bind {
808            self.check_bind(n, name, &start, true)?;
809            // bind_class binds every method in the same transaction (the store's rule).
810            for (mname, _) in &methods {
811                self.check_bind(n, mname, &start, false)?;
812            }
813            self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
814            for (mname, fh) in &methods {
815                self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
816            }
817        }
818        self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
819        self.ops.push(Op::Class {
820            name: name.to_string(),
821            start,
822            hash,
823            bytes: def.bytes,
824            bind,
825            methods: methods.into_iter().map(|(m, _)| m).collect(),
826        });
827        Ok(())
828    }
829
830    /// One bind a line would make: rule A, the other kind, the namespace, and the latest binding.
831    fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
832        if !self.backfill && start < self.ctx.now {
833            return Err(PublishError::typed(
834                line,
835                PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
836            ));
837        }
838        let err = |d: String| PublishError::at(line, d);
839        if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
840            let kind = if class { "function" } else { "class" };
841            return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
842        }
843        suspended(&mut self.reads, self.store, host_of(name), line)?;
844        if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
845            if start <= latest.as_str() {
846                return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
847            }
848        }
849        self.binds.push(Bind { line, name: name.into() });
850        Ok(())
851    }
852
853    /// Plans the cell lines of a chunk (CONTRACT-perf H1): each is prepared in file order up to its
854    /// receipt; the receipts are signed on threads; the store is read once for the chunk; then each
855    /// is checked in file order. The first failure is the chunk's, and a line that fails before
856    /// signing is reported after the lines before it are checked.
857    fn plan_cells(&mut self, lines: Vec<(usize, Value)>) -> Result<(), PublishError> {
858        if lines.is_empty() {
859            return Ok(());
860        }
861        let mut pending = Vec::with_capacity(lines.len());
862        let mut failed = None;
863        for (n, v) in &lines {
864            match self.prepare_cell(*n, v) {
865                Ok(p) => pending.push(p),
866                Err(e) => {
867                    failed = Some(e);
868                    break;
869                }
870            }
871        }
872        let (key, host) = (self.key, self.ctx.host);
873        let signed = parallel_owned(pending, |p| p.sign(key, host));
874        let ids: Vec<&str> = signed.iter().flatten().map(|s| s.id.as_str()).collect();
875        self.reads.prefetch_latest(self.store, &ids);
876        let hashes: Vec<String> = signed
877            .iter()
878            .flatten()
879            .filter(|s| !s.has_tx)
880            .filter_map(|s| self.reads.latest_of.get(&s.id).cloned().flatten())
881            .map(|(_, h)| h)
882            .collect();
883        let hashes: Vec<&str> = hashes.iter().map(String::as_str).collect();
884        self.reads.prefetch_receipts(self.store, &hashes);
885        for s in signed {
886            self.check_cell(s?)?;
887        }
888        match failed {
889            Some(e) => Err(e),
890            None => Ok(()),
891        }
892    }
893
894    /// A cell line up to its receipt, as the checks of a line take them in order: the fields, the
895    /// instants, the function as of `tx`, the descriptor, the arguments and the value (and its blob).
896    fn prepare_cell(&mut self, n: usize, v: &Value) -> Result<Pending, PublishError> {
897        let err = |d: String| PublishError::at(n, d);
898        let cell_v = v.get("cell").ok_or_else(|| err("a cell without fn".into()))?;
899        let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
900        let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
901        let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
902        let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
903        let (licence, summary) = (required("licence")?, required("summary")?);
904        // tx and asof default to the run's clock now, each on its own (H4).
905        let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
906        let has_tx = field("tx").is_some();
907        let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
908        if tx > asof {
909            return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
910        }
911        let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
912        if asof > issued {
913            return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
914        }
915        let fn_hash = function_at(&mut self.reads, self.store, &self.sim, name, &tx)
916            .map_err(err)?
917            .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
918        let d = function_bytes_of(&mut self.reads, self.store, &self.sim, &fn_hash)
919            .map_err(err)?
920            .ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
921        let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
922        let mut args = Args::new();
923        for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
924            args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
925        }
926        let valid: Args = args
927            .iter()
928            .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
929            .map(|(k, a)| (k.clone(), a.clone()))
930            .collect();
931        let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
932        let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
933            Some(file) => {
934                let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
935                let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
936                let sha = sha256_hex(&bytes);
937                let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
938                    .map_err(|e| err(e.to_string()))?;
939                (scalar, Some(bytes))
940            }
941            None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
942        };
943        let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
944        Ok(Pending {
945            line: n,
946            ns: ns.to_string(),
947            cell,
948            value,
949            valid,
950            tx,
951            asof,
952            issued,
953            has_tx,
954            licence,
955            summary,
956            blob,
957        })
958    }
959
960    /// The checks of a signed cell line, in file order: the skip rule of a line without `tx`, then
961    /// the version rule. Its write is planned.
962    fn check_cell(&mut self, s: Signed) -> Result<(), PublishError> {
963        let line = s.line;
964        let err = |d: String| PublishError::at(line, d);
965        // A line without tx takes the run's clock, so a re-run at a later clock would sign the same
966        // fact again as a new version. Re-running a file changes nothing: when the cell's latest
967        // version is the store's and states the same thing (type, value, valid, licence, summary),
968        // the line is skipped. A changed value is a new version, a correction.
969        if !s.has_tx {
970            if let Some((_, latest)) = cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
971                if let Some(old) = self.reads.receipt(self.store, &latest) {
972                    let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
973                    let new = json::parse(&s.receipt).map_err(|e| err(e.to_string()))?;
974                    if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
975                        return Ok(());
976                    }
977                }
978            }
979        }
980        let version = match cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
981            Some((latest_tx, _)) if latest_tx.as_str() >= s.tx.as_str() => {
982                if cell_version_asof(self.store, &self.sim, &s.id, &s.tx).map_err(err)?.as_deref()
983                    != Some(s.hash.as_str())
984                {
985                    return Err(err(format!("{} has a version at or after {} that is not this receipt", s.id, s.tx)));
986                }
987                false
988            }
989            _ => {
990                self.sim.versions.entry(s.id.clone()).or_default().push((s.tx.clone(), s.hash.clone()));
991                true
992            }
993        };
994        self.ops.push(Op::Cell { tx: s.tx, id: s.id, receipt: s.receipt, blob: s.blob, version });
995        Ok(())
996    }
997}
998
999/// The stored descriptor of a function line's signature.
1000fn stored_value(signature: &Value) -> Value {
1001    Value::Object(vec![
1002        ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
1003        ("inputs".into(), Value::Array(vec![])),
1004        ("signature".into(), signature.clone()),
1005    ])
1006}
1007
1008/// The backfill's checks (since 0.3.0 with store 0.4.2): no name the file binds has a binding, and
1009/// no namespace of one has any binding (`has_bindings_under`).
1010fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
1011    for b in binds {
1012        let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
1013        if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
1014            return Err(refuse(format!("{} has a binding", b.name)));
1015        }
1016        let ns = namespace_of(&b.name);
1017        if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
1018            return Err(refuse(format!("{ns} has bindings")));
1019        }
1020    }
1021    Ok(())
1022}
1023
1024/// Appends a run's line to the backfill log (created with mode 0600).
1025fn append_log(log: &Path, written: &Written) -> Result<(), String> {
1026    let line = Value::Object(vec![
1027        ("at".into(), s(&SystemClock.now())),
1028        ("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
1029        ("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
1030    ])
1031    .to_jcs();
1032    let mut opts = std::fs::OpenOptions::new();
1033    opts.create(true).append(true);
1034    #[cfg(unix)]
1035    {
1036        use std::os::unix::fs::OpenOptionsExt;
1037        opts.mode(0o600);
1038    }
1039    let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
1040    let mut bytes = line;
1041    bytes.push(b'\n');
1042    f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
1043}
1044
1045/// Pass two: the writes the first pass planned, in order. Nothing here can refuse a planned
1046/// write, so a refused file has written nothing. Consecutive cell writes go in `put_cells` calls
1047/// of up to `CHUNK` on `lanes` lanes, their blobs put first (CONTRACT-perf H2, H5).
1048fn write<S: Store + ?Sized>(
1049    store: &S,
1050    ops: &[Op],
1051    cells: impl Fn(&S, &[Op], &mut Published) -> Result<(), String>,
1052) -> Result<(Published, Written), PublishError> {
1053    let mut out = Published::default();
1054    let mut written = Written::default();
1055    let err = |d: String| PublishError::at(0, d);
1056    let mut i = 0;
1057    while i < ops.len() {
1058        match &ops[i] {
1059            Op::Descriptor { hash, bytes, counted } => {
1060                let existed = store.descriptor(hash).is_some();
1061                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1062                if *counted && !existed {
1063                    out.descriptors += 1;
1064                }
1065            }
1066            Op::Function { name, start, hash, bytes, bind } => {
1067                store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1068                if *bind {
1069                    store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1070                    out.functions += 1;
1071                    written.bound(name, start);
1072                }
1073            }
1074            Op::Class { name, start, hash, bytes, bind, methods } => {
1075                store.put_class(bytes).map_err(|e| err(e.to_string()))?;
1076                if *bind {
1077                    store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1078                    out.classes += 1;
1079                    written.bound(name, start);
1080                    for m in methods {
1081                        written.bound(m, start);
1082                    }
1083                }
1084            }
1085            Op::Cell { .. } => {
1086                let mut j = i;
1087                while j < ops.len() && j - i < CHUNK && matches!(ops[j], Op::Cell { .. }) {
1088                    j += 1;
1089                }
1090                cells(store, &ops[i..j], &mut out).map_err(err)?;
1091                i = j;
1092                continue;
1093            }
1094        }
1095        i += 1;
1096    }
1097    Ok((out, written))
1098}
1099
1100/// A run of cell writes (up to `CHUNK`) on one lane: their blobs first, then the receipts and
1101/// versions in file order in `put_cells` calls of up to `CHUNK` (CONTRACT-perf H2). No `Sync`
1102/// bound: this is what [`publish_stored_with`] runs.
1103fn write_cells_one_lane<S: Store + ?Sized>(store: &S, group: &[Op], out: &mut Published) -> Result<(), String> {
1104    let items = put_blobs_and_items(store, group, out)?;
1105    let failed = AtomicBool::new(false);
1106    let ks: Vec<usize> = (0..items.len()).collect();
1107    if let Some((_, detail)) = put_lane(store, &items, &ks, &failed) {
1108        return Err(detail);
1109    }
1110    count_cells(group, &items, out);
1111    Ok(())
1112}
1113
1114/// A run of cell writes (up to `CHUNK`): their blobs first, then their receipts and versions on
1115/// `lanes` lanes (CONTRACT-perf H5). A cell's id picks its lane (its first hex digit, mod `lanes`),
1116/// so all writes of one cell share a lane, in file order. A lane's `put_cells` calls take up to
1117/// `CHUNK` of its items. When one fails, no lane starts another call, and the error returned is
1118/// that of the earliest failing item in file order. With one lane, as [`write_cells_one_lane`].
1119fn write_cells<S: Store + Sync + ?Sized>(
1120    store: &S,
1121    group: &[Op],
1122    lanes: usize,
1123    out: &mut Published,
1124) -> Result<(), String> {
1125    if lanes <= 1 {
1126        return write_cells_one_lane(store, group, out);
1127    }
1128    let items = put_blobs_and_items(store, group, out)?;
1129    let mut by_lane: Vec<Vec<usize>> = vec![Vec::new(); lanes];
1130    for (k, op) in group.iter().enumerate() {
1131        if let Op::Cell { id, .. } = op {
1132            by_lane[lane_of(id, lanes)].push(k);
1133        }
1134    }
1135    let failed = AtomicBool::new(false);
1136    let failures: Vec<(usize, String)> = std::thread::scope(|scope| {
1137        let handles: Vec<_> = by_lane
1138            .iter()
1139            .map(|ks| {
1140                let (items, failed) = (&items, &failed);
1141                scope.spawn(move || put_lane(store, items, ks, failed))
1142            })
1143            .collect();
1144        handles.into_iter().filter_map(|h| h.join().unwrap_or_else(|p| std::panic::resume_unwind(p))).collect()
1145    });
1146    if let Some((_, detail)) = failures.into_iter().min_by_key(|(k, _)| *k) {
1147        return Err(detail);
1148    }
1149    count_cells(group, &items, out);
1150    Ok(())
1151}
1152
1153/// The blobs of a run, put in order (counted in `out`), and the run's cell writes.
1154fn put_blobs_and_items<'a, S: Store + ?Sized>(
1155    store: &S,
1156    group: &'a [Op],
1157    out: &mut Published,
1158) -> Result<Vec<CellItem<'a>>, String> {
1159    for op in group {
1160        if let Op::Cell { blob: Some(bytes), .. } = op {
1161            store.put_blob(bytes).map_err(|e| e.to_string())?;
1162            out.blobs += 1;
1163            out.blob_bytes += bytes.len() as u64;
1164        }
1165    }
1166    Ok(group
1167        .iter()
1168        .filter_map(|op| match op {
1169            Op::Cell { tx, id, receipt, version, .. } => {
1170                Some((receipt.as_slice(), if *version { Some((id.as_str(), tx.as_str())) } else { None }))
1171            }
1172            _ => None,
1173        })
1174        .collect())
1175}
1176
1177/// Counts a written run of cells in `out`.
1178fn count_cells(group: &[Op], items: &[CellItem<'_>], out: &mut Published) {
1179    out.cells += group.len();
1180    out.receipts += group.len();
1181    out.versions += items.iter().filter(|(_, v)| v.is_some()).count();
1182}
1183
1184/// The lane of a cell id: its first hex digit, mod `lanes` (CONTRACT-perf H5).
1185fn lane_of(id: &str, lanes: usize) -> usize {
1186    id.bytes().next().and_then(|b| (b as char).to_digit(16)).unwrap_or(0) as usize % lanes
1187}
1188
1189/// A cell write of a run: its receipt's bytes and its version (cell id, tx), if any.
1190type CellItem<'a> = (&'a [u8], Option<(&'a str, &'a str)>);
1191
1192/// One lane's writes: its items (indices into `items`, in file order) in `put_cells` calls of up to
1193/// `CHUNK`. Returns the index of the first item of the call that failed, and its error. No call
1194/// starts once `failed` is set.
1195fn put_lane<S: Store + ?Sized>(
1196    store: &S,
1197    items: &[CellItem<'_>],
1198    ks: &[usize],
1199    failed: &AtomicBool,
1200) -> Option<(usize, String)> {
1201    for call in ks.chunks(CHUNK) {
1202        if failed.load(Ordering::SeqCst) {
1203            return None;
1204        }
1205        let writes: Vec<CellWrite<'_>> = call.iter().map(|&k| CellWrite::new(items[k].0, items[k].1)).collect();
1206        if let Err(e) = store.put_cells(&writes) {
1207            failed.store(true, Ordering::SeqCst);
1208            return Some((call[0], e.to_string()));
1209        }
1210    }
1211    None
1212}
1213
1214/// The receipt JSON of a signed receipt: its payload's members and `sig`.
1215pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
1216    let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
1217    if let Ok(Value::Object(m)) = json::parse(r.payload()) {
1218        members.extend(m);
1219    }
1220    Value::Object(members).to_jcs()
1221}
1222
1223/// Publishes the file at `path` into `store` as the mesh `host` signing with `key` (see the
1224/// module documentation). The mesh's own namespaces are `host` alone: see [`publish_stored_in`].
1225/// The clock is the system clock, and there is no backfill. The cells are written on one lane;
1226/// see [`publish_stored_lanes`] for the parallel lanes.
1227pub fn publish_stored<S: Store + ?Sized>(
1228    store: &S,
1229    key: &SigningKey,
1230    host: &str,
1231    path: &Path,
1232) -> Result<Published, PublishError> {
1233    publish_stored_in(store, key, host, &[host.to_string()], path)
1234}
1235
1236/// As [`publish_stored`], for a mesh that owns the namespaces (hosts) `owned` (`Config::namespaces`):
1237/// a class line is accepted for a class under one of them, and refused with `E_CLASS_OWNER` (in the
1238/// [`PublishError::detail`]) under any other. Since 0.2.2. The system clock, and no backfill.
1239pub fn publish_stored_in<S: Store + ?Sized>(
1240    store: &S,
1241    key: &SigningKey,
1242    host: &str,
1243    owned: &[String],
1244    path: &Path,
1245) -> Result<Published, PublishError> {
1246    publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
1247}
1248
1249/// As [`publish_stored_in`], with the run's clock and backfill (CONTRACT-1.8b H1, rule A). The
1250/// cells are written in file order on one lane, as in 0.3.2 (CONTRACT-perf H2).
1251///
1252/// Errors: a line refused by its checks, or a binding whose start is earlier than the clock
1253/// ([`PublishErrorKind::StartBeforeNow`]), or a backfill refused ([`PublishErrorKind::Backfill`]).
1254/// Nothing is written when the first pass refuses (CONTRACT-1.7a H2).
1255pub fn publish_stored_with<S: Store + ?Sized>(
1256    store: &S,
1257    key: &SigningKey,
1258    host: &str,
1259    owned: &[String],
1260    path: &Path,
1261    opts: &PublishOptions,
1262) -> Result<Published, PublishError> {
1263    publish_planned(store, key, host, owned, path, opts, |s, ops| {
1264        write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1265    })
1266}
1267
1268/// As [`publish_stored_with`], with the cells written on parallel lanes (CONTRACT-perf H5, H6):
1269/// `DATUMESH_PUBLISH_LANES` (a positive integer) if set, else `min(4, available_parallelism())`.
1270/// The same receipts, versions and rows as one lane; only the order of the inserts across
1271/// different cells differs. The variable is read once per run, before the file is read.
1272pub fn publish_stored_lanes<S: Store + Sync + ?Sized>(
1273    store: &S,
1274    key: &SigningKey,
1275    host: &str,
1276    owned: &[String],
1277    path: &Path,
1278    opts: &PublishOptions,
1279) -> Result<Published, PublishError> {
1280    let lanes = publish_lanes()?;
1281    publish_planned(store, key, host, owned, path, opts, |s, ops| {
1282        write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1283    })
1284}
1285
1286/// The first pass (the planner) and the second pass (`write_ops`) of a publish.
1287fn publish_planned<S: Store + ?Sized>(
1288    store: &S,
1289    key: &SigningKey,
1290    host: &str,
1291    owned: &[String],
1292    path: &Path,
1293    opts: &PublishOptions,
1294    write_ops: impl FnOnce(&S, &[Op]) -> Result<(Published, Written), PublishError>,
1295) -> Result<Published, PublishError> {
1296    let lines = read_lines(path)?;
1297    // The key's start: the earliest `issued`.
1298    let key_from = store
1299        .namespace(host)
1300        .ok()
1301        .flatten()
1302        .and_then(|n| json::parse(&n.keydoc).ok())
1303        .and_then(|d| match d.get("keys") {
1304            Some(Value::Array(ks)) => ks
1305                .iter()
1306                .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
1307                .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
1308            _ => None,
1309        })
1310        .ok_or_else(|| {
1311            PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
1312        })?;
1313    let now = opts.clock.now();
1314    let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
1315    let mut planner = Planner {
1316        store,
1317        key,
1318        ctx,
1319        backfill: opts.backfill.is_some(),
1320        sim: Simulation::default(),
1321        reads: Reads::default(),
1322        ops: Vec::new(),
1323        binds: Vec::new(),
1324    };
1325    planner.run(&lines)?;
1326    let Planner { ops, binds, .. } = planner;
1327    if opts.backfill.is_some() {
1328        backfill_check(store, &binds)?;
1329    }
1330    let (out, written) = write_ops(store, &ops)?;
1331    if let Some(log) = &opts.backfill {
1332        if out.functions + out.classes + out.versions > 0 {
1333            append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
1334        }
1335    }
1336    Ok(out)
1337}