1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
62use std::io::{BufRead, Write};
63use std::path::{Path, PathBuf};
64use std::sync::atomic::{AtomicBool, Ordering};
65use std::sync::Arc;
66
67use datumesh_core::json::{self, Value};
68use datumesh_core::{
69 sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey,
70 Source, Type,
71};
72use datumesh_engine::{Clock, SystemClock};
73use datumesh_formula::definition::{Callee, Callees, ClassCallee};
74use datumesh_store::{Binding, CellWrite, FunctionTest, Namespace, Store};
75
76use crate::testrun::{self, run_test, Overlay, Ran, RowWrite, TestSpec};
77
78use crate::config::ConfigError;
79
80const END: &str = "9999-12-31T23:59:59.999999Z";
82
83const CHUNK: usize = 1_000;
85
86const LANES_VAR: &str = "DATUMESH_PUBLISH_LANES";
88
89fn publish_lanes() -> Result<usize, PublishError> {
92 let bad = |detail: String| PublishError::at(0, ConfigError::new(LANES_VAR, detail).to_string());
93 match std::env::var(LANES_VAR) {
94 Err(std::env::VarError::NotPresent) => Ok(std::thread::available_parallelism().map_or(1, |n| n.get()).min(4)),
95 Ok(v) => match v.parse::<usize>() {
96 Ok(n) if n > 0 => Ok(n),
97 _ => Err(bad(format!("must be a positive integer, not {v:?}"))),
98 },
99 Err(e) => Err(bad(e.to_string())),
100 }
101}
102
103#[non_exhaustive]
105#[derive(Clone, Debug, Default, PartialEq, Eq)]
106pub struct Published {
107 pub functions: usize,
109 pub classes: usize,
111 pub cells: usize,
113 pub receipts: usize,
115 pub versions: usize,
117 pub blobs: usize,
119 pub blob_bytes: u64,
121 pub descriptors: u64,
123}
124
125#[derive(Clone)]
127pub struct PublishOptions {
128 pub clock: Arc<dyn Clock + Send + Sync>,
131 pub backfill: Option<PathBuf>,
135}
136
137impl PublishOptions {
138 pub fn system() -> PublishOptions {
140 PublishOptions { clock: Arc::new(SystemClock), backfill: None }
141 }
142}
143
144#[non_exhaustive]
146#[derive(Clone, Debug, PartialEq, Eq)]
147pub enum PublishErrorKind {
148 Line,
150 StartBeforeNow {
152 name: String,
154 start: String,
156 now: String,
158 },
159 Backfill(String),
161 OwnedVectorName(String),
163 TestsMissing(String),
166 TestsFailing(String),
169}
170
171impl std::fmt::Display for PublishErrorKind {
172 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
173 match self {
174 PublishErrorKind::Line => f.write_str("refused"),
175 PublishErrorKind::StartBeforeNow { name, start, now } => {
176 write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
177 }
178 PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
179 PublishErrorKind::OwnedVectorName(name) => {
180 write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
181 }
182 PublishErrorKind::TestsMissing(text) => write!(f, "E_TESTS_MISSING: {text}"),
183 PublishErrorKind::TestsFailing(text) => write!(f, "E_TESTS_FAILING: {text}"),
184 }
185 }
186}
187
188#[derive(Clone, Debug, PartialEq, Eq)]
190pub struct PublishError {
191 pub line: usize,
193 pub detail: String,
195 pub kind: PublishErrorKind,
197}
198
199impl PublishError {
200 fn at(line: usize, detail: impl Into<String>) -> PublishError {
201 PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
202 }
203
204 pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
205 PublishError { line, detail: kind.to_string(), kind }
206 }
207}
208
209impl std::fmt::Display for PublishError {
210 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
211 write!(f, "line {}: {}", self.line, self.detail)
212 }
213}
214
215impl std::error::Error for PublishError {}
216
217fn s(v: &str) -> Value {
218 Value::String(v.to_string())
219}
220
221fn namespace_of(name: &str) -> &str {
223 name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
224}
225
226fn host_of(name: &str) -> &str {
228 name.split('/').next().unwrap_or("")
229}
230
231fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
234 let host_of_name = name.split('/').next().unwrap_or("");
235 if owned.iter().any(|o| o == host_of_name) {
236 namespace_of(name)
237 } else {
238 host
239 }
240}
241
242fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
244 match v.get("bound") {
245 None => Ok(now.to_string()),
246 Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
247 }
248}
249
250#[derive(Default)]
253struct Simulation {
254 functions: BTreeMap<String, Vec<(String, String)>>,
256 classes: BTreeMap<String, Vec<(String, String)>>,
258 descriptors: BTreeMap<String, Vec<u8>>,
260 class_bytes: BTreeMap<String, Vec<u8>>,
262 versions: BTreeMap<String, Vec<(String, String)>>,
264}
265
266impl Simulation {
267 fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
269 binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
270 }
271}
272
273struct Callers<'a, S: Store + ?Sized> {
275 store: &'a S,
276 sim: Option<&'a Simulation>,
277}
278
279impl<S: Store + ?Sized> Callers<'_, S> {
280 fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
281 if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
282 return Ok(Some(b.clone()));
283 }
284 Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
285 }
286
287 fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
288 if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
289 return Ok(Some(b.clone()));
290 }
291 self.store.class(hash).map_err(|e| e.to_string())
292 }
293}
294
295impl<S: Store + ?Sized> Callees for Callers<'_, S> {
296 fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
297 let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
298 let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
299 let (fn_hash, start) = match (stored, simmed) {
300 (Some(b), Some((t, h))) if t >= b.tx => (h, t),
301 (Some(b), _) => (b.fn_hash, b.tx),
302 (None, Some((t, h))) => (h, t),
303 (None, None) => return Ok(None),
304 };
305 let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
306 let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
307 Ok(Some(Callee { fn_hash, start, descriptor }))
308 }
309
310 fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
311 let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
312 let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
313 let (class_hash, start) = match (stored, simmed) {
314 (Some(b), Some((t, h))) if t >= b.tx => (h, t),
315 (Some(b), _) => (b.class_hash, b.tx),
316 (None, Some((t, h))) => (h, t),
317 (None, None) => return Ok(None),
318 };
319 let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
320 let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
321 Ok(Some(ClassCallee { class_hash, start, descriptor }))
322 }
323
324 fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
325 let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
326 ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
327 }
328
329 fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
330 let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
331 json::parse(&bytes).map(Some).map_err(|e| e.to_string())
332 }
333}
334
335#[derive(Default)]
340struct Reads {
341 bindings: HashMap<(String, String), Result<Option<Binding>, String>>,
342 descriptors: HashMap<String, Option<Vec<u8>>>,
343 namespaces: HashMap<String, Result<Option<Namespace>, String>>,
344 latest_of: HashMap<String, Option<(String, String)>>,
346 receipt_of: HashMap<String, Option<Vec<u8>>>,
348}
349
350impl Reads {
351 fn binding<S: Store + ?Sized>(&mut self, store: &S, name: &str, at: &str) -> Result<Option<Binding>, String> {
352 let key = (name.to_string(), at.to_string());
353 if let Some(r) = self.bindings.get(&key) {
354 return r.clone();
355 }
356 let r = store.resolve_binding(name, at).map_err(|e| e.to_string());
357 self.bindings.insert(key, r.clone());
358 r
359 }
360
361 fn descriptor<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
362 if let Some(b) = self.descriptors.get(hash) {
363 return b.clone();
364 }
365 let b = store.descriptor(hash).map(|b| b.into_owned());
366 self.descriptors.insert(hash.to_string(), b.clone());
367 b
368 }
369
370 fn namespace<S: Store + ?Sized>(&mut self, store: &S, host: &str) -> Result<Option<Namespace>, String> {
371 if let Some(r) = self.namespaces.get(host) {
372 return r.clone();
373 }
374 let r = store.namespace(host).map_err(|e| e.to_string());
375 self.namespaces.insert(host.to_string(), r.clone());
376 r
377 }
378
379 fn latest<S: Store + ?Sized>(&mut self, store: &S, id: &str) -> Result<Option<(String, String)>, String> {
381 if let Some(l) = self.latest_of.get(id) {
382 return Ok(l.clone());
383 }
384 store.latest(id).map(|v| v.map(|v| (v.tx, v.receipt_hash))).map_err(|e| e.to_string())
385 }
386
387 fn receipt<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
389 if let Some(b) = self.receipt_of.get(hash) {
390 return b.clone();
391 }
392 store.receipt(hash).map(|b| b.into_owned())
393 }
394
395 fn prefetch_latest<S: Store + ?Sized>(&mut self, store: &S, ids: &[&str]) {
397 let mut seen = HashSet::new();
398 let missing: Vec<&str> =
399 ids.iter().copied().filter(|id| !self.latest_of.contains_key(*id) && seen.insert(*id)).collect();
400 if missing.is_empty() {
401 return;
402 }
403 if let Ok(found) = store.latest_many(&missing) {
404 for (id, v) in missing.iter().zip(found) {
405 self.latest_of.insert(id.to_string(), v.map(|v| (v.tx, v.receipt_hash)));
406 }
407 }
408 }
409
410 fn prefetch_receipts<S: Store + ?Sized>(&mut self, store: &S, hashes: &[&str]) {
412 let mut seen = HashSet::new();
413 let missing: Vec<&str> =
414 hashes.iter().copied().filter(|h| !self.receipt_of.contains_key(*h) && seen.insert(*h)).collect();
415 if missing.is_empty() {
416 return;
417 }
418 if let Ok(found) = store.receipts_many(&missing) {
419 for (h, b) in missing.iter().zip(found) {
420 self.receipt_of.insert(h.to_string(), b);
421 }
422 }
423 }
424}
425
426fn parallel_owned<T: Send, R: Send>(items: Vec<T>, f: impl Fn(T) -> R + Sync) -> Vec<R> {
428 let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(items.len());
429 if threads <= 1 {
430 return items.into_iter().map(f).collect();
431 }
432 let size = items.len().div_ceil(threads);
433 let mut parts: Vec<Vec<T>> = Vec::with_capacity(threads);
434 let mut rest = items.into_iter();
435 loop {
436 let part: Vec<T> = rest.by_ref().take(size).collect();
437 if part.is_empty() {
438 break;
439 }
440 parts.push(part);
441 }
442 std::thread::scope(|s| {
443 let handles: Vec<_> = parts
444 .into_iter()
445 .map(|part| {
446 let f = &f;
447 s.spawn(move || part.into_iter().map(f).collect::<Vec<R>>())
448 })
449 .collect();
450 handles.into_iter().flat_map(|h| h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect()
451 })
452}
453
454fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
456 if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
457 return Ok(Some(h));
458 }
459 store.resolve_name(name, END).map_err(|e| e.to_string())
460}
461
462fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
464 if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
465 return Ok(Some(h));
466 }
467 store.resolve_class(name, END).map_err(|e| e.to_string())
468}
469
470fn function_at<S: Store + ?Sized>(
472 reads: &mut Reads,
473 store: &S,
474 sim: &Simulation,
475 name: &str,
476 at: &str,
477) -> Result<Option<String>, String> {
478 let stored = reads.binding(store, name, at)?;
479 let simmed = Simulation::latest(&sim.functions, name, at);
480 Ok(match (stored, simmed) {
481 (Some(b), Some((t, h))) if t >= b.tx => Some(h),
482 (Some(b), _) => Some(b.fn_hash),
483 (None, Some((_, h))) => Some(h),
484 (None, None) => None,
485 })
486}
487
488fn latest_bind<S: Store + ?Sized>(
490 store: &S,
491 sim: &Simulation,
492 name: &str,
493 class: bool,
494) -> Result<Option<String>, String> {
495 let stored: Option<String> = if class {
496 store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
497 } else {
498 store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
499 };
500 let binds = if class { &sim.classes } else { &sim.functions };
501 let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
502 Ok(match (stored, planned) {
503 (Some(a), Some(b)) => Some(a.max(b)),
504 (a, b) => a.or(b),
505 })
506}
507
508fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
510 if class {
511 Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
512 } else {
513 Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
514 || sim.classes.contains_key(name))
515 }
516}
517
518fn cell_latest<S: Store + ?Sized>(
520 reads: &mut Reads,
521 store: &S,
522 sim: &Simulation,
523 id: &str,
524) -> Result<Option<(String, String)>, String> {
525 let stored = reads.latest(store, id)?;
526 let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
527 Ok(match (stored, planned) {
528 (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
529 (a, b) => a.or(b),
530 })
531}
532
533fn cell_version_asof<S: Store + ?Sized>(
535 store: &S,
536 sim: &Simulation,
537 id: &str,
538 at: &str,
539) -> Result<Option<String>, String> {
540 let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
541 let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
542 let latest = match (stored, planned) {
543 (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
544 (a, b) => a.or(b),
545 };
546 Ok(latest.map(|x| x.1))
547}
548
549fn function_bytes_of<S: Store + ?Sized>(
551 reads: &mut Reads,
552 store: &S,
553 sim: &Simulation,
554 hash: &str,
555) -> Result<Option<Vec<u8>>, String> {
556 if let Some(b) = sim.descriptors.get(hash) {
557 return Ok(Some(b.clone()));
558 }
559 Ok(reads.descriptor(store, hash))
560}
561
562fn suspended<S: Store + ?Sized>(reads: &mut Reads, store: &S, host: &str, line: usize) -> Result<(), PublishError> {
564 match reads.namespace(store, host) {
565 Ok(Some(n)) if n.suspended_at.is_some() => {
566 Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
567 }
568 Ok(_) => Ok(()),
569 Err(e) => Err(PublishError::at(line, e)),
570 }
571}
572
573struct Bind {
575 line: usize,
576 name: String,
577}
578
579#[derive(Default)]
581struct Written {
582 namespaces: BTreeSet<String>,
583 starts: BTreeSet<String>,
584}
585
586impl Written {
587 fn bound(&mut self, name: &str, start: &str) {
588 self.namespaces.insert(namespace_of(name).to_string());
589 self.starts.insert(start.to_string());
590 }
591}
592
593struct Ctx<'a> {
595 host: &'a str,
596 owned: &'a [String],
597 key_from: &'a str,
599 now: &'a str,
601 base: &'a Path,
603}
604
605enum Op {
607 Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
609 Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
611 Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
613 Cell { tx: String, id: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
615 Catalog {
617 entry: datumesh_store::CatalogEntry,
618 dictionaries: Vec<(String, Vec<datumesh_store::KeyValue>)>,
619 open_keys: Vec<String>,
620 },
621}
622
623fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
625 let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
626 let mut out = Vec::new();
627 for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
628 let n = i + 1;
629 let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
630 if !line.trim().is_empty() {
631 out.push((n, line));
632 }
633 }
634 Ok(out)
635}
636
637fn index_cell_values<S: Store + ?Sized>(store: &S, ops: &[Op]) -> Result<(), String> {
641 let mut seen = BTreeSet::new();
642 for op in ops {
643 if let Op::Cell { receipt, .. } = op {
644 if let Ok(r) = datumesh_core::Receipt::from_json(receipt) {
645 cell_string_values(&r, &mut seen);
646 }
647 }
648 }
649 store_cell_values(store, seen)
650}
651
652pub(crate) fn cell_string_values(r: &datumesh_core::Receipt, out: &mut BTreeSet<(String, String, String)>) {
654 let cell = r.cell();
655 for (key, arg) in &cell.args {
656 if let datumesh_core::Arg::Value(datumesh_core::Scalar::String(v)) = arg {
657 if !v.is_empty() && v.len() <= 1024 {
658 out.insert((cell.function_name(), key.clone(), v.clone()));
659 }
660 }
661 }
662}
663
664pub(crate) fn dictionary_limit() -> u64 {
668 std::env::var("DATUMESH_DICTIONARY_MAX")
669 .ok()
670 .and_then(|v| v.trim().parse::<u64>().ok())
671 .filter(|n| *n >= 1)
672 .unwrap_or(1000)
673}
674
675pub(crate) fn store_cell_values<S: Store + ?Sized>(
680 store: &S,
681 seen: BTreeSet<(String, String, String)>,
682) -> Result<(), String> {
683 let mut groups: BTreeMap<(String, String), Vec<String>> = BTreeMap::new();
684 for (owner, key, value) in seen {
685 groups.entry((owner, key)).or_default().push(value);
686 }
687 let limit = dictionary_limit();
688 for ((owner, key), values) in groups {
689 if store.key_is_open(&owner, &key).map_err(|e| e.to_string())? {
690 continue;
691 }
692 let fresh: Vec<String> =
693 values.into_iter().filter(|v| !crate::find::has_value(store, &owner, &key, v)).collect();
694 if fresh.is_empty() {
695 continue;
696 }
697 let have = store.key_value_count(&owner, &key).map_err(|e| e.to_string())?;
698 if have + fresh.len() as u64 > limit {
699 store.mark_key_open(&owner, &key).map_err(|e| e.to_string())?;
700 continue;
701 }
702 let kvs: Vec<datumesh_store::KeyValue> = fresh
703 .into_iter()
704 .map(|value| datumesh_store::KeyValue { value, label: String::new(), aliases: Vec::new() })
705 .collect();
706 store.upsert_key_values(&owner, &key, &kvs).map_err(|e| e.to_string())?;
707 }
708 Ok(())
709}
710
711fn row_name(own: Option<usize>, spec: &TestSpec, group: &str) -> String {
714 match own {
715 Some(_) => spec.function.clone(),
716 None => group.to_string(),
717 }
718}
719
720fn is_cell_line(v: &Value) -> bool {
722 v.get("class").and_then(Value::as_str).is_none()
723 && v.get("descriptor").is_none()
724 && v.get("function").and_then(Value::as_str).is_none()
725 && v.get("formula").and_then(Value::as_str).is_none()
726 && v.get("cell").is_some()
727}
728
729struct Pending {
731 line: usize,
732 ns: String,
733 cell: Cell,
734 value: Scalar,
735 valid: Args,
736 tx: String,
737 asof: String,
738 issued: String,
739 has_tx: bool,
740 licence: String,
741 summary: String,
742 blob: Option<Vec<u8>>,
743 source: Option<Source>,
745}
746
747fn source_of_line(v: &Value) -> Result<Source, String> {
751 let Value::Object(m) = v else { return Err("source is an object".into()) };
752 for (k, x) in m {
753 if !matches!(k.as_str(), "name" | "url" | "retrieved" | "sha256" | "takeover") {
754 return Err(format!("source has no member {k}"));
755 }
756 if !matches!(x, Value::String(_)) {
757 return Err(format!("source.{k} is a string"));
758 }
759 }
760 let text = |k: &str| -> Result<&str, String> {
761 v.get(k).and_then(Value::as_str).ok_or_else(|| format!("source has no {k}"))
762 };
763 let mut out = Source::new(text("name")?, text("url")?, text("retrieved")?);
764 if let Some(h) = v.get("sha256").and_then(Value::as_str) {
765 out = out.with_sha256(h);
766 }
767 if let Some(t) = v.get("takeover").and_then(Value::as_str) {
768 out = out.with_takeover(t);
769 }
770 Ok(out)
771}
772
773impl Pending {
774 fn sign(self, key: &SigningKey, host: &str) -> Result<Signed, PublishError> {
776 let line = self.line;
777 let err = move |d: String| PublishError::at(line, d);
778 let b = ReceiptBuilder::new()
779 .cell(self.cell.clone())
780 .value(self.value)
781 .valid(self.valid)
782 .asof(self.asof.as_str())
783 .tx(self.tx.as_str())
784 .issued(self.issued.as_str())
785 .publisher(self.ns.as_str())
786 .signer_host(host)
787 .licence(self.licence.as_str())
788 .summary(self.summary.as_str());
789 let b = match self.source {
790 Some(s) => b.source(s),
791 None => b,
792 };
793 let r = b.sign(key).map_err(|e| err(e.to_string()))?;
794 let receipt = receipt_json(&r);
795 let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
796 let id = self.cell.id();
797 Ok(Signed { line, has_tx: self.has_tx, tx: self.tx, id, hash, receipt, blob: self.blob })
798 }
799}
800
801struct Signed {
803 line: usize,
804 has_tx: bool,
805 tx: String,
806 id: String,
807 hash: String,
808 receipt: Vec<u8>,
809 blob: Option<Vec<u8>>,
810}
811
812struct Planner<'a, S: Store + ?Sized> {
814 store: &'a S,
815 key: &'a SigningKey,
816 ctx: Ctx<'a>,
817 backfill: bool,
818 sim: Simulation,
819 reads: Reads,
820 ops: Vec<Op>,
821 binds: Vec<Bind>,
822 tests: Vec<(usize, TestSpec)>,
824 formulas: Vec<(usize, String, String)>,
826}
827
828impl<S: Store + ?Sized> Planner<'_, S> {
829 fn callers(&self) -> Callers<'_, S> {
830 Callers { store: self.store, sim: Some(&self.sim) }
831 }
832
833 fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
836 let mut cells: Vec<(usize, Value)> = Vec::new();
837 for (n, text) in lines {
838 let n = *n;
839 match json::parse(text.as_bytes()).map_err(|e| PublishError::at(n, e.to_string())) {
840 Ok(v) if is_cell_line(&v) => {
841 if let Some(name) = v.get("cell").and_then(|c| c.get("fn")).and_then(Value::as_str) {
843 crate::reserved::check(self.ctx.host, name, None).map_err(|e| PublishError::at(n, e))?;
844 }
845 cells.push((n, v));
846 if cells.len() == CHUNK {
847 self.plan_cells(std::mem::take(&mut cells))?;
848 }
849 }
850 other => {
851 self.plan_cells(std::mem::take(&mut cells))?;
854 self.plan_line(n, &other?)?;
855 }
856 }
857 }
858 self.plan_cells(cells)
859 }
860
861 fn plan_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
863 let err = |d: String| PublishError::at(n, d);
864 let named =
867 v.get("class").or_else(|| v.get("function")).or_else(|| v.get("formula")).or_else(|| v.get("catalog"));
868 if let Some(name) = named.and_then(Value::as_str) {
869 crate::reserved::check(self.ctx.host, name, None).map_err(err)?;
870 }
871 if let Some(name) = v.get("class").and_then(Value::as_str) {
872 self.class_line(n, name, v)?;
873 } else if let Some(d) = v.get("descriptor") {
874 self.descriptor_line(n, d)?;
875 } else if let Some(name) = v.get("function").and_then(Value::as_str) {
876 let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
877 let start = bound_of(v, n, self.ctx.now)?;
878 let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
879 self.function(n, name, start, fd.hash(), fd.canonical())?;
880 } else if v.get("catalog").is_some() {
881 self.catalog_line(n, v)?;
882 } else if let Some(name) = v.get("formula").and_then(Value::as_str) {
883 let source =
884 v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
885 let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
886 let start = bound_of(v, n, self.ctx.now)?;
887 let def = datumesh_formula::definition::define_function(
888 name,
889 source,
890 sig,
891 &Default::default(),
892 &self.callers(),
893 &start,
894 )
895 .map_err(|e| err(format!("define {name}: {e}")))?;
896 let hash = def.fn_hash.clone();
897 self.function(n, name, start, def.fn_hash, def.bytes)?;
898 self.formulas.push((n, name.to_string(), hash));
899 } else if v.get("test").is_some() {
900 let spec = testrun::parse_test(v).map_err(err)?;
901 self.tests.push((n, spec));
902 } else {
903 return Err(err("neither a function nor a cell".into()));
904 }
905 Ok(())
906 }
907
908 fn catalog_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
912 let err = |d: String| PublishError::at(n, d);
913 let d = crate::catalog::parse(v, "catalog").map_err(err)?;
914 let host = host_of(&d.name);
915 if host != self.ctx.host && !self.ctx.owned.iter().any(|o| o == host) {
916 return Err(err(format!("{} is not under a host this mesh owns", d.name)));
917 }
918 let hash = match d.kind {
919 datumesh_store::CatalogKind::Class => current_class(self.store, &self.sim, &d.name).map_err(err)?,
920 _ => current_function(self.store, &self.sim, &d.name).map_err(err)?,
921 };
922 if hash.is_none() {
923 return Err(err(format!("{} is bound nowhere", d.name)));
924 }
925 let entry = d.entry(hash, datumesh_store::CatalogSource::Publisher, Some(self.key.key_id().to_string()));
926 self.ops.push(Op::Catalog { entry, dictionaries: d.dictionary, open_keys: d.open_keys });
927 Ok(())
928 }
929
930 fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
932 let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
933 if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
934 return Err(PublishError::at(
935 n,
936 "a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
937 ));
938 }
939 let (hash, bytes) = (fd.hash(), fd.canonical());
940 self.sim.descriptors.insert(hash.clone(), bytes.clone());
941 self.ops.push(Op::Descriptor { hash, bytes, counted: true });
942 Ok(())
943 }
944
945 fn function(
947 &mut self,
948 n: usize,
949 name: &str,
950 start: String,
951 hash: String,
952 bytes: Vec<u8>,
953 ) -> Result<(), PublishError> {
954 self.sim.descriptors.insert(hash.clone(), bytes.clone());
955 let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
956 let bind = current.as_deref() != Some(hash.as_str());
957 if bind {
958 self.check_bind(n, name, &start, false)?;
959 self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
960 }
961 self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
962 Ok(())
963 }
964
965 fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
966 let err = |d: String| PublishError::at(n, d);
967 let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
968 let start = bound_of(v, n, self.ctx.now)?;
969 let def = datumesh_formula::definition::define_class(
970 name,
971 &descriptor.to_jcs(),
972 definer(name, self.ctx.host, self.ctx.owned),
973 &self.callers(),
974 &start,
975 )
976 .map_err(|e| err(format!("define {name}: {e}")))?;
977 suspended(&mut self.reads, self.store, host_of(&def.descriptor.steward.namespace), n)?;
981 let hash = def.class_hash.clone();
982 let methods: Vec<(String, String)> =
983 def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
984 let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
985 if bind {
986 self.check_bind(n, name, &start, true)?;
987 for (mname, _) in &methods {
989 self.check_bind(n, mname, &start, false)?;
990 }
991 self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
992 for (mname, fh) in &methods {
993 self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
994 }
995 }
996 self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
997 self.ops.push(Op::Class {
998 name: name.to_string(),
999 start,
1000 hash,
1001 bytes: def.bytes,
1002 bind,
1003 methods: methods.into_iter().map(|(m, _)| m).collect(),
1004 });
1005 Ok(())
1006 }
1007
1008 fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
1010 if !self.backfill && start < self.ctx.now {
1011 return Err(PublishError::typed(
1012 line,
1013 PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
1014 ));
1015 }
1016 let err = |d: String| PublishError::at(line, d);
1017 if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
1018 let kind = if class { "function" } else { "class" };
1019 return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
1020 }
1021 suspended(&mut self.reads, self.store, host_of(name), line)?;
1022 if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
1023 if start <= latest.as_str() {
1024 return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
1025 }
1026 }
1027 self.binds.push(Bind { line, name: name.into() });
1028 Ok(())
1029 }
1030
1031 fn plan_cells(&mut self, lines: Vec<(usize, Value)>) -> Result<(), PublishError> {
1036 if lines.is_empty() {
1037 return Ok(());
1038 }
1039 let mut pending = Vec::with_capacity(lines.len());
1040 let mut failed = None;
1041 for (n, v) in &lines {
1042 match self.prepare_cell(*n, v) {
1043 Ok(p) => pending.push(p),
1044 Err(e) => {
1045 failed = Some(e);
1046 break;
1047 }
1048 }
1049 }
1050 let (key, host) = (self.key, self.ctx.host);
1051 let signed = parallel_owned(pending, |p| p.sign(key, host));
1052 let ids: Vec<&str> = signed.iter().flatten().map(|s| s.id.as_str()).collect();
1053 self.reads.prefetch_latest(self.store, &ids);
1054 let hashes: Vec<String> = signed
1055 .iter()
1056 .flatten()
1057 .filter(|s| !s.has_tx)
1058 .filter_map(|s| self.reads.latest_of.get(&s.id).cloned().flatten())
1059 .map(|(_, h)| h)
1060 .collect();
1061 let hashes: Vec<&str> = hashes.iter().map(String::as_str).collect();
1062 self.reads.prefetch_receipts(self.store, &hashes);
1063 for s in signed {
1064 self.check_cell(s?)?;
1065 }
1066 match failed {
1067 Some(e) => Err(e),
1068 None => Ok(()),
1069 }
1070 }
1071
1072 fn prepare_cell(&mut self, n: usize, v: &Value) -> Result<Pending, PublishError> {
1075 let err = |d: String| PublishError::at(n, d);
1076 let cell_v = v.get("cell").ok_or_else(|| err("a cell without fn".into()))?;
1077 let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
1078 let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
1079 let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
1080 let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
1081 let (licence, summary) = (required("licence")?, required("summary")?);
1082 let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
1084 let has_tx = field("tx").is_some();
1085 let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
1086 if tx > asof {
1087 return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
1088 }
1089 let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
1090 if asof > issued {
1091 return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
1092 }
1093 let fn_hash = function_at(&mut self.reads, self.store, &self.sim, name, &tx)
1094 .map_err(err)?
1095 .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
1096 let d = function_bytes_of(&mut self.reads, self.store, &self.sim, &fn_hash)
1097 .map_err(err)?
1098 .ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
1099 let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
1100 let mut args = Args::new();
1101 for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
1102 args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
1103 }
1104 let valid: Args = args
1105 .iter()
1106 .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
1107 .map(|(k, a)| (k.clone(), a.clone()))
1108 .collect();
1109 let source = match v.get("source") {
1110 Some(sv) => Some(source_of_line(sv).map_err(err)?),
1111 None => None,
1112 };
1113 let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
1114 let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
1115 Some(file) => {
1116 let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
1117 let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
1118 let sha = sha256_hex(&bytes);
1119 let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
1120 .map_err(|e| err(e.to_string()))?;
1121 (scalar, Some(bytes))
1122 }
1123 None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
1124 };
1125 let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
1126 Ok(Pending {
1127 line: n,
1128 ns: ns.to_string(),
1129 cell,
1130 value,
1131 valid,
1132 tx,
1133 asof,
1134 issued,
1135 has_tx,
1136 licence,
1137 summary,
1138 blob,
1139 source,
1140 })
1141 }
1142
1143 fn check_cell(&mut self, s: Signed) -> Result<(), PublishError> {
1146 let line = s.line;
1147 let err = |d: String| PublishError::at(line, d);
1148 if !s.has_tx {
1153 if let Some((_, latest)) = cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1154 if let Some(old) = self.reads.receipt(self.store, &latest) {
1155 let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
1156 let new = json::parse(&s.receipt).map_err(|e| err(e.to_string()))?;
1157 if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
1158 return Ok(());
1159 }
1160 }
1161 }
1162 }
1163 let version = match cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1164 Some((latest_tx, _)) if latest_tx.as_str() >= s.tx.as_str() => {
1165 if cell_version_asof(self.store, &self.sim, &s.id, &s.tx).map_err(err)?.as_deref()
1166 != Some(s.hash.as_str())
1167 {
1168 return Err(err(format!("{} has a version at or after {} that is not this receipt", s.id, s.tx)));
1169 }
1170 false
1171 }
1172 _ => {
1173 self.sim.versions.entry(s.id.clone()).or_default().push((s.tx.clone(), s.hash.clone()));
1174 true
1175 }
1176 };
1177 self.ops.push(Op::Cell { tx: s.tx, id: s.id, receipt: s.receipt, blob: s.blob, version });
1178 Ok(())
1179 }
1180}
1181
1182fn stored_value(signature: &Value) -> Value {
1184 Value::Object(vec![
1185 ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
1186 ("inputs".into(), Value::Array(vec![])),
1187 ("signature".into(), signature.clone()),
1188 ])
1189}
1190
1191fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
1194 for b in binds {
1195 let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
1196 if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
1197 return Err(refuse(format!("{} has a binding", b.name)));
1198 }
1199 let ns = namespace_of(&b.name);
1200 if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
1201 return Err(refuse(format!("{ns} has bindings")));
1202 }
1203 }
1204 Ok(())
1205}
1206
1207fn append_log(log: &Path, written: &Written) -> Result<(), String> {
1209 let line = Value::Object(vec![
1210 ("at".into(), s(&SystemClock.now())),
1211 ("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
1212 ("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
1213 ])
1214 .to_jcs();
1215 let mut opts = std::fs::OpenOptions::new();
1216 opts.create(true).append(true);
1217 #[cfg(unix)]
1218 {
1219 use std::os::unix::fs::OpenOptionsExt;
1220 opts.mode(0o600);
1221 }
1222 let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
1223 let mut bytes = line;
1224 bytes.push(b'\n');
1225 f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
1226}
1227
1228fn write<S: Store + ?Sized>(
1232 store: &S,
1233 ops: &[Op],
1234 cells: impl Fn(&S, &[Op], &mut Published) -> Result<(), String>,
1235) -> Result<(Published, Written), PublishError> {
1236 let mut out = Published::default();
1237 let mut written = Written::default();
1238 let err = |d: String| PublishError::at(0, d);
1239 let mut i = 0;
1240 while i < ops.len() {
1241 match &ops[i] {
1242 Op::Descriptor { hash, bytes, counted } => {
1243 let existed = store.descriptor(hash).is_some();
1244 store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1245 if *counted && !existed {
1246 out.descriptors += 1;
1247 }
1248 }
1249 Op::Function { name, start, hash, bytes, bind } => {
1250 store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1251 if *bind {
1252 store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1253 out.functions += 1;
1254 written.bound(name, start);
1255 }
1256 }
1257 Op::Class { name, start, hash, bytes, bind, methods } => {
1258 store.put_class(bytes).map_err(|e| err(e.to_string()))?;
1259 if *bind {
1260 store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1261 out.classes += 1;
1262 written.bound(name, start);
1263 for m in methods {
1264 written.bound(m, start);
1265 }
1266 }
1267 }
1268 Op::Catalog { entry, dictionaries, open_keys } => {
1269 crate::catalog::store_publisher(store, entry, dictionaries, open_keys).map_err(err)?;
1270 }
1271 Op::Cell { .. } => {
1272 let mut j = i;
1273 while j < ops.len() && j - i < CHUNK && matches!(ops[j], Op::Cell { .. }) {
1274 j += 1;
1275 }
1276 cells(store, &ops[i..j], &mut out).map_err(err)?;
1277 i = j;
1278 continue;
1279 }
1280 }
1281 i += 1;
1282 }
1283 Ok((out, written))
1284}
1285
1286fn write_cells_one_lane<S: Store + ?Sized>(store: &S, group: &[Op], out: &mut Published) -> Result<(), String> {
1290 let items = put_blobs_and_items(store, group, out)?;
1291 let failed = AtomicBool::new(false);
1292 let ks: Vec<usize> = (0..items.len()).collect();
1293 if let Some((_, detail)) = put_lane(store, &items, &ks, &failed) {
1294 return Err(detail);
1295 }
1296 count_cells(group, &items, out);
1297 Ok(())
1298}
1299
1300fn write_cells<S: Store + Sync + ?Sized>(
1306 store: &S,
1307 group: &[Op],
1308 lanes: usize,
1309 out: &mut Published,
1310) -> Result<(), String> {
1311 if lanes <= 1 {
1312 return write_cells_one_lane(store, group, out);
1313 }
1314 let items = put_blobs_and_items(store, group, out)?;
1315 let mut by_lane: Vec<Vec<usize>> = vec![Vec::new(); lanes];
1316 for (k, op) in group.iter().enumerate() {
1317 if let Op::Cell { id, .. } = op {
1318 by_lane[lane_of(id, lanes)].push(k);
1319 }
1320 }
1321 let failed = AtomicBool::new(false);
1322 let failures: Vec<(usize, String)> = std::thread::scope(|scope| {
1323 let handles: Vec<_> = by_lane
1324 .iter()
1325 .map(|ks| {
1326 let (items, failed) = (&items, &failed);
1327 scope.spawn(move || put_lane(store, items, ks, failed))
1328 })
1329 .collect();
1330 handles.into_iter().filter_map(|h| h.join().unwrap_or_else(|p| std::panic::resume_unwind(p))).collect()
1331 });
1332 if let Some((_, detail)) = failures.into_iter().min_by_key(|(k, _)| *k) {
1333 return Err(detail);
1334 }
1335 count_cells(group, &items, out);
1336 Ok(())
1337}
1338
1339fn put_blobs_and_items<'a, S: Store + ?Sized>(
1341 store: &S,
1342 group: &'a [Op],
1343 out: &mut Published,
1344) -> Result<Vec<CellItem<'a>>, String> {
1345 for op in group {
1346 if let Op::Cell { blob: Some(bytes), .. } = op {
1347 store.put_blob(bytes).map_err(|e| e.to_string())?;
1348 out.blobs += 1;
1349 out.blob_bytes += bytes.len() as u64;
1350 }
1351 }
1352 Ok(group
1353 .iter()
1354 .filter_map(|op| match op {
1355 Op::Cell { tx, id, receipt, version, .. } => {
1356 Some((receipt.as_slice(), if *version { Some((id.as_str(), tx.as_str())) } else { None }))
1357 }
1358 _ => None,
1359 })
1360 .collect())
1361}
1362
1363fn count_cells(group: &[Op], items: &[CellItem<'_>], out: &mut Published) {
1365 out.cells += group.len();
1366 out.receipts += group.len();
1367 out.versions += items.iter().filter(|(_, v)| v.is_some()).count();
1368}
1369
1370fn lane_of(id: &str, lanes: usize) -> usize {
1372 id.bytes().next().and_then(|b| (b as char).to_digit(16)).unwrap_or(0) as usize % lanes
1373}
1374
1375type CellItem<'a> = (&'a [u8], Option<(&'a str, &'a str)>);
1377
1378fn put_lane<S: Store + ?Sized>(
1382 store: &S,
1383 items: &[CellItem<'_>],
1384 ks: &[usize],
1385 failed: &AtomicBool,
1386) -> Option<(usize, String)> {
1387 for call in ks.chunks(CHUNK) {
1388 if failed.load(Ordering::SeqCst) {
1389 return None;
1390 }
1391 let writes: Vec<CellWrite<'_>> = call.iter().map(|&k| CellWrite::new(items[k].0, items[k].1)).collect();
1392 if let Err(e) = store.put_cells(&writes) {
1393 failed.store(true, Ordering::SeqCst);
1394 return Some((call[0], e.to_string()));
1395 }
1396 }
1397 None
1398}
1399
1400pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
1402 let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
1403 if let Ok(Value::Object(m)) = json::parse(r.payload()) {
1404 members.extend(m);
1405 }
1406 Value::Object(members).to_jcs()
1407}
1408
1409pub fn publish_stored<S: Store + ?Sized>(
1414 store: &S,
1415 key: &SigningKey,
1416 host: &str,
1417 path: &Path,
1418) -> Result<Published, PublishError> {
1419 publish_stored_in(store, key, host, &[host.to_string()], path)
1420}
1421
1422pub fn publish_stored_in<S: Store + ?Sized>(
1426 store: &S,
1427 key: &SigningKey,
1428 host: &str,
1429 owned: &[String],
1430 path: &Path,
1431) -> Result<Published, PublishError> {
1432 publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
1433}
1434
1435pub fn publish_stored_with<S: Store + ?Sized>(
1442 store: &S,
1443 key: &SigningKey,
1444 host: &str,
1445 owned: &[String],
1446 path: &Path,
1447 opts: &PublishOptions,
1448) -> Result<Published, PublishError> {
1449 publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1450 write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1451 })
1452}
1453
1454pub fn publish_stored_checked<S: Store + ?Sized>(
1461 store: &S,
1462 key: &SigningKey,
1463 host: &str,
1464 owned: &[String],
1465 path: &Path,
1466 opts: &PublishOptions,
1467) -> Result<Published, PublishError> {
1468 publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1469 write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1470 })
1471}
1472
1473pub fn publish_stored_lanes<S: Store + Sync + ?Sized>(
1478 store: &S,
1479 key: &SigningKey,
1480 host: &str,
1481 owned: &[String],
1482 path: &Path,
1483 opts: &PublishOptions,
1484) -> Result<Published, PublishError> {
1485 let lanes = publish_lanes()?;
1486 publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1487 write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1488 })
1489}
1490
1491fn overlay_of(ops: &[Op]) -> Result<Overlay, PublishError> {
1493 let mut overlay = Overlay::default();
1494 for op in ops {
1495 match op {
1496 Op::Function { name, start, hash, bytes, bind } => {
1497 overlay.descriptors.insert(hash.clone(), bytes.clone());
1498 if *bind {
1499 overlay.functions.entry(name.clone()).or_default().push((start.clone(), hash.clone()));
1500 }
1501 }
1502 Op::Descriptor { hash, bytes, .. } => {
1503 overlay.descriptors.insert(hash.clone(), bytes.clone());
1504 }
1505 Op::Cell { receipt, .. } => {
1506 overlay.cell(receipt).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1507 }
1508 Op::Class { .. } | Op::Catalog { .. } => {}
1509 }
1510 }
1511 Ok(overlay)
1512}
1513
1514#[allow(clippy::too_many_arguments)]
1522pub(crate) fn check_tests<S: Store + ?Sized>(
1523 store: &S,
1524 overlay: &Overlay,
1525 tests: &[(usize, TestSpec)],
1526 formulas: &[(usize, String, String)],
1527 host: &str,
1528 now: &str,
1529 checked: bool,
1530) -> Result<Vec<RowWrite>, PublishError> {
1531 let mut file_tests: Vec<(usize, TestSpec, String)> = Vec::new();
1532 for (n, spec) in tests {
1533 let at = spec.asof.clone().unwrap_or_else(|| now.to_string());
1534 match testrun::binding_of(store, overlay, &spec.function, &at) {
1535 Ok(Some((hash, _))) => file_tests.push((*n, spec.clone(), hash)),
1536 Ok(None) => {
1537 return Err(PublishError::at(
1538 *n,
1539 format!("a test of {}: no function of that name is bound", spec.function),
1540 ))
1541 }
1542 Err(e) => return Err(PublishError::at(*n, e)),
1543 }
1544 }
1545 let mut groups: Vec<(usize, String, String)> = formulas.to_vec();
1547 for (n, spec, hash) in &file_tests {
1548 if !groups.iter().any(|g| &g.2 == hash) {
1549 groups.push((*n, spec.function.clone(), hash.clone()));
1550 }
1551 }
1552 let mut rows = Vec::new();
1553 for (line, name, hash) in &groups {
1554 let own: Vec<(usize, TestSpec)> =
1555 file_tests.iter().filter(|t| &t.2 == hash).map(|t| (t.0, t.1.clone())).collect();
1556 let all_held = testrun::held_of_name(store, hash, name, now).map_err(|e| PublishError::at(*line, e))?;
1558 if !checked && own.is_empty() && all_held.is_empty() {
1559 continue;
1560 }
1561 let mut run = testrun::Run::new(store, overlay, now);
1562 let built = run.built(name, hash).map_err(|e| PublishError::at(*line, format!("{name}: {}", e.0)))?;
1563 if built.compiled.is_some() && !formulas.iter().any(|f| &f.2 == hash) {
1566 continue;
1567 }
1568 let written: BTreeSet<&str> = overlay.cells.iter().filter(|c| &c.name == name).map(|c| c.id.as_str()).collect();
1571 let held: Vec<(FunctionTest, TestSpec, String)> = all_held
1572 .into_iter()
1573 .filter(|(_, _, cell)| built.compiled.is_some() || written.contains(cell.as_str()))
1574 .collect();
1575 let mut specs: Vec<(Option<usize>, TestSpec)> = own.into_iter().map(|(n, s)| (Some(n), s)).collect();
1576 for (_, spec, _) in held {
1577 if !specs.iter().any(|(_, s)| s.case == spec.case) {
1578 specs.push((None, spec));
1579 }
1580 }
1581 let ran: Vec<Ran> = specs.iter().map(|(_, s)| run_test(store, overlay, now, s)).collect();
1582 let Some(compiled) = built.compiled.as_ref() else {
1583 for ((own_n, spec), r) in specs.iter().zip(&ran) {
1585 let (result, detail) = testrun::result_of(spec, r);
1586 rows.push(RowWrite {
1587 fn_hash: hash.clone(),
1588 case: spec.case.clone(),
1589 jcs: spec.jcs.clone(),
1590 signed_by: host.to_string(),
1591 result,
1592 detail,
1593 branches_total: 0,
1594 branches_covered: 0,
1595 fn_name: row_name(*own_n, spec, name),
1596 });
1597 }
1598 continue;
1599 };
1600 if checked {
1601 let mut failing: Vec<(usize, String)> = Vec::new();
1602 for ((n, spec), r) in specs.iter().zip(&ran) {
1603 if let Some(n) = n {
1604 let (result, detail) = testrun::result_of(spec, r);
1605 if result != "pass" {
1606 failing.push((*n, format!("{name}: test {} {detail}", spec.case)));
1607 }
1608 }
1609 }
1610 failing.sort_by_key(|f| f.0);
1611 if let Some((n, detail)) = failing.into_iter().next() {
1612 return Err(PublishError::typed(n, PublishErrorKind::TestsFailing(detail)));
1613 }
1614 let refs: Vec<&Ran> = ran.iter().collect();
1615 let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1616 if !cov.uncovered.is_empty() || !cov.unexercised.is_empty() {
1617 let mut text = format!("{name} is missing tests:");
1618 if !cov.uncovered.is_empty() {
1619 let cases: Vec<&str> = cov.uncovered.iter().map(|c| c.0.as_str()).collect();
1620 text += &format!(" uncovered cases {};", cases.join(", "));
1621 }
1622 if !cov.unexercised.is_empty() {
1623 text += &format!(" unexercised branches {}", cov.unexercised.join(", "));
1624 }
1625 return Err(PublishError::typed(
1626 *line,
1627 PublishErrorKind::TestsMissing(text.trim_end_matches(';').to_string()),
1628 ));
1629 }
1630 }
1631 let refs: Vec<&Ran> = ran.iter().collect();
1632 let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1633 for ((own_n, spec), r) in specs.iter().zip(&ran) {
1634 let (result, detail) = testrun::result_of(spec, r);
1635 rows.push(RowWrite {
1636 fn_hash: hash.clone(),
1637 case: spec.case.clone(),
1638 jcs: spec.jcs.clone(),
1639 signed_by: host.to_string(),
1640 result,
1641 detail,
1642 branches_total: cov.branches,
1643 branches_covered: cov.covered,
1644 fn_name: row_name(*own_n, spec, name),
1645 });
1646 }
1647 }
1648 Ok(rows)
1649}
1650
1651pub fn publish_stored_checked_lanes<S: Store + Sync + ?Sized>(
1654 store: &S,
1655 key: &SigningKey,
1656 host: &str,
1657 owned: &[String],
1658 path: &Path,
1659 opts: &PublishOptions,
1660) -> Result<Published, PublishError> {
1661 let lanes = publish_lanes()?;
1662 publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1663 write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1664 })
1665}
1666
1667#[allow(clippy::too_many_arguments)]
1669fn publish_planned<S: Store + ?Sized>(
1670 store: &S,
1671 key: &SigningKey,
1672 host: &str,
1673 owned: &[String],
1674 path: &Path,
1675 opts: &PublishOptions,
1676 checked: bool,
1677 write_ops: impl FnOnce(&S, &[Op]) -> Result<(Published, Written), PublishError>,
1678) -> Result<Published, PublishError> {
1679 let lines = read_lines(path)?;
1680 let key_from = store
1682 .namespace(host)
1683 .ok()
1684 .flatten()
1685 .and_then(|n| json::parse(&n.keydoc).ok())
1686 .and_then(|d| match d.get("keys") {
1687 Some(Value::Array(ks)) => ks
1688 .iter()
1689 .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
1690 .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
1691 _ => None,
1692 })
1693 .ok_or_else(|| {
1694 PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
1695 })?;
1696 let now = opts.clock.now();
1697 let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
1698 let mut planner = Planner {
1699 store,
1700 key,
1701 ctx,
1702 backfill: opts.backfill.is_some(),
1703 sim: Simulation::default(),
1704 reads: Reads::default(),
1705 ops: Vec::new(),
1706 binds: Vec::new(),
1707 tests: Vec::new(),
1708 formulas: Vec::new(),
1709 };
1710 planner.run(&lines)?;
1711 let Planner { ops, binds, tests, formulas, .. } = planner;
1712 if opts.backfill.is_some() {
1713 backfill_check(store, &binds)?;
1714 }
1715 let overlay = overlay_of(&ops)?;
1718 let rows = check_tests(store, &overlay, &tests, &formulas, host, &now, checked)?;
1719 let (out, written) = write_ops(store, &ops)?;
1720 index_cell_values(store, &ops).map_err(|e| PublishError::at(0, format!("catalogue: {e}")))?;
1721 testrun::apply_rows(store, &rows, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1722 let rebound: Vec<String> = ops
1723 .iter()
1724 .filter_map(|op| match op {
1725 Op::Function { name, bind: true, .. } => Some(name.clone()),
1726 _ => None,
1727 })
1728 .collect();
1729 testrun::rerun_readers(store, &rebound, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1730 let rebound: BTreeSet<String> = rebound.into_iter().collect();
1732 let written_cells: Vec<(String, String)> = overlay.cells.iter().map(|c| (c.name.clone(), c.id.clone())).collect();
1733 testrun::rerun_stored_tests(store, &rebound, &written_cells, &now)
1734 .map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1735 if let Some(log) = &opts.backfill {
1736 if out.functions + out.classes + out.versions > 0 {
1737 append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
1738 }
1739 }
1740 Ok(out)
1741}