1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
59use std::io::{BufRead, Write};
60use std::path::{Path, PathBuf};
61use std::sync::atomic::{AtomicBool, Ordering};
62use std::sync::Arc;
63
64use datumesh_core::json::{self, Value};
65use datumesh_core::{
66 sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey, Type,
67};
68use datumesh_engine::{Clock, SystemClock};
69use datumesh_formula::definition::{Callee, Callees, ClassCallee};
70use datumesh_store::{Binding, CellWrite, Namespace, Store};
71
72use crate::testrun::{self, run_test, Overlay, Ran, RowWrite, TestSpec};
73
74use crate::config::ConfigError;
75
76const END: &str = "9999-12-31T23:59:59.999999Z";
78
79const CHUNK: usize = 1_000;
81
82const LANES_VAR: &str = "DATUMESH_PUBLISH_LANES";
84
85fn publish_lanes() -> Result<usize, PublishError> {
88 let bad = |detail: String| PublishError::at(0, ConfigError::new(LANES_VAR, detail).to_string());
89 match std::env::var(LANES_VAR) {
90 Err(std::env::VarError::NotPresent) => Ok(std::thread::available_parallelism().map_or(1, |n| n.get()).min(4)),
91 Ok(v) => match v.parse::<usize>() {
92 Ok(n) if n > 0 => Ok(n),
93 _ => Err(bad(format!("must be a positive integer, not {v:?}"))),
94 },
95 Err(e) => Err(bad(e.to_string())),
96 }
97}
98
99#[non_exhaustive]
101#[derive(Clone, Debug, Default, PartialEq, Eq)]
102pub struct Published {
103 pub functions: usize,
105 pub classes: usize,
107 pub cells: usize,
109 pub receipts: usize,
111 pub versions: usize,
113 pub blobs: usize,
115 pub blob_bytes: u64,
117 pub descriptors: u64,
119}
120
121#[derive(Clone)]
123pub struct PublishOptions {
124 pub clock: Arc<dyn Clock + Send + Sync>,
127 pub backfill: Option<PathBuf>,
131}
132
133impl PublishOptions {
134 pub fn system() -> PublishOptions {
136 PublishOptions { clock: Arc::new(SystemClock), backfill: None }
137 }
138}
139
140#[non_exhaustive]
142#[derive(Clone, Debug, PartialEq, Eq)]
143pub enum PublishErrorKind {
144 Line,
146 StartBeforeNow {
148 name: String,
150 start: String,
152 now: String,
154 },
155 Backfill(String),
157 OwnedVectorName(String),
159 TestsMissing(String),
162 TestsFailing(String),
165}
166
167impl std::fmt::Display for PublishErrorKind {
168 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
169 match self {
170 PublishErrorKind::Line => f.write_str("refused"),
171 PublishErrorKind::StartBeforeNow { name, start, now } => {
172 write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
173 }
174 PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
175 PublishErrorKind::OwnedVectorName(name) => {
176 write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
177 }
178 PublishErrorKind::TestsMissing(text) => write!(f, "E_TESTS_MISSING: {text}"),
179 PublishErrorKind::TestsFailing(text) => write!(f, "E_TESTS_FAILING: {text}"),
180 }
181 }
182}
183
184#[derive(Clone, Debug, PartialEq, Eq)]
186pub struct PublishError {
187 pub line: usize,
189 pub detail: String,
191 pub kind: PublishErrorKind,
193}
194
195impl PublishError {
196 fn at(line: usize, detail: impl Into<String>) -> PublishError {
197 PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
198 }
199
200 pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
201 PublishError { line, detail: kind.to_string(), kind }
202 }
203}
204
205impl std::fmt::Display for PublishError {
206 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
207 write!(f, "line {}: {}", self.line, self.detail)
208 }
209}
210
211impl std::error::Error for PublishError {}
212
213fn s(v: &str) -> Value {
214 Value::String(v.to_string())
215}
216
217fn namespace_of(name: &str) -> &str {
219 name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
220}
221
222fn host_of(name: &str) -> &str {
224 name.split('/').next().unwrap_or("")
225}
226
227fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
230 let host_of_name = name.split('/').next().unwrap_or("");
231 if owned.iter().any(|o| o == host_of_name) {
232 namespace_of(name)
233 } else {
234 host
235 }
236}
237
238fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
240 match v.get("bound") {
241 None => Ok(now.to_string()),
242 Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
243 }
244}
245
246#[derive(Default)]
249struct Simulation {
250 functions: BTreeMap<String, Vec<(String, String)>>,
252 classes: BTreeMap<String, Vec<(String, String)>>,
254 descriptors: BTreeMap<String, Vec<u8>>,
256 class_bytes: BTreeMap<String, Vec<u8>>,
258 versions: BTreeMap<String, Vec<(String, String)>>,
260}
261
262impl Simulation {
263 fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
265 binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
266 }
267}
268
269struct Callers<'a, S: Store + ?Sized> {
271 store: &'a S,
272 sim: Option<&'a Simulation>,
273}
274
275impl<S: Store + ?Sized> Callers<'_, S> {
276 fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
277 if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
278 return Ok(Some(b.clone()));
279 }
280 Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
281 }
282
283 fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
284 if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
285 return Ok(Some(b.clone()));
286 }
287 self.store.class(hash).map_err(|e| e.to_string())
288 }
289}
290
291impl<S: Store + ?Sized> Callees for Callers<'_, S> {
292 fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
293 let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
294 let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
295 let (fn_hash, start) = match (stored, simmed) {
296 (Some(b), Some((t, h))) if t >= b.tx => (h, t),
297 (Some(b), _) => (b.fn_hash, b.tx),
298 (None, Some((t, h))) => (h, t),
299 (None, None) => return Ok(None),
300 };
301 let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
302 let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
303 Ok(Some(Callee { fn_hash, start, descriptor }))
304 }
305
306 fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
307 let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
308 let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
309 let (class_hash, start) = match (stored, simmed) {
310 (Some(b), Some((t, h))) if t >= b.tx => (h, t),
311 (Some(b), _) => (b.class_hash, b.tx),
312 (None, Some((t, h))) => (h, t),
313 (None, None) => return Ok(None),
314 };
315 let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
316 let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
317 Ok(Some(ClassCallee { class_hash, start, descriptor }))
318 }
319
320 fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
321 let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
322 ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
323 }
324
325 fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
326 let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
327 json::parse(&bytes).map(Some).map_err(|e| e.to_string())
328 }
329}
330
331#[derive(Default)]
336struct Reads {
337 bindings: HashMap<(String, String), Result<Option<Binding>, String>>,
338 descriptors: HashMap<String, Option<Vec<u8>>>,
339 namespaces: HashMap<String, Result<Option<Namespace>, String>>,
340 latest_of: HashMap<String, Option<(String, String)>>,
342 receipt_of: HashMap<String, Option<Vec<u8>>>,
344}
345
346impl Reads {
347 fn binding<S: Store + ?Sized>(&mut self, store: &S, name: &str, at: &str) -> Result<Option<Binding>, String> {
348 let key = (name.to_string(), at.to_string());
349 if let Some(r) = self.bindings.get(&key) {
350 return r.clone();
351 }
352 let r = store.resolve_binding(name, at).map_err(|e| e.to_string());
353 self.bindings.insert(key, r.clone());
354 r
355 }
356
357 fn descriptor<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
358 if let Some(b) = self.descriptors.get(hash) {
359 return b.clone();
360 }
361 let b = store.descriptor(hash).map(|b| b.into_owned());
362 self.descriptors.insert(hash.to_string(), b.clone());
363 b
364 }
365
366 fn namespace<S: Store + ?Sized>(&mut self, store: &S, host: &str) -> Result<Option<Namespace>, String> {
367 if let Some(r) = self.namespaces.get(host) {
368 return r.clone();
369 }
370 let r = store.namespace(host).map_err(|e| e.to_string());
371 self.namespaces.insert(host.to_string(), r.clone());
372 r
373 }
374
375 fn latest<S: Store + ?Sized>(&mut self, store: &S, id: &str) -> Result<Option<(String, String)>, String> {
377 if let Some(l) = self.latest_of.get(id) {
378 return Ok(l.clone());
379 }
380 store.latest(id).map(|v| v.map(|v| (v.tx, v.receipt_hash))).map_err(|e| e.to_string())
381 }
382
383 fn receipt<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
385 if let Some(b) = self.receipt_of.get(hash) {
386 return b.clone();
387 }
388 store.receipt(hash).map(|b| b.into_owned())
389 }
390
391 fn prefetch_latest<S: Store + ?Sized>(&mut self, store: &S, ids: &[&str]) {
393 let mut seen = HashSet::new();
394 let missing: Vec<&str> =
395 ids.iter().copied().filter(|id| !self.latest_of.contains_key(*id) && seen.insert(*id)).collect();
396 if missing.is_empty() {
397 return;
398 }
399 if let Ok(found) = store.latest_many(&missing) {
400 for (id, v) in missing.iter().zip(found) {
401 self.latest_of.insert(id.to_string(), v.map(|v| (v.tx, v.receipt_hash)));
402 }
403 }
404 }
405
406 fn prefetch_receipts<S: Store + ?Sized>(&mut self, store: &S, hashes: &[&str]) {
408 let mut seen = HashSet::new();
409 let missing: Vec<&str> =
410 hashes.iter().copied().filter(|h| !self.receipt_of.contains_key(*h) && seen.insert(*h)).collect();
411 if missing.is_empty() {
412 return;
413 }
414 if let Ok(found) = store.receipts_many(&missing) {
415 for (h, b) in missing.iter().zip(found) {
416 self.receipt_of.insert(h.to_string(), b);
417 }
418 }
419 }
420}
421
422fn parallel_owned<T: Send, R: Send>(items: Vec<T>, f: impl Fn(T) -> R + Sync) -> Vec<R> {
424 let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(items.len());
425 if threads <= 1 {
426 return items.into_iter().map(f).collect();
427 }
428 let size = items.len().div_ceil(threads);
429 let mut parts: Vec<Vec<T>> = Vec::with_capacity(threads);
430 let mut rest = items.into_iter();
431 loop {
432 let part: Vec<T> = rest.by_ref().take(size).collect();
433 if part.is_empty() {
434 break;
435 }
436 parts.push(part);
437 }
438 std::thread::scope(|s| {
439 let handles: Vec<_> = parts
440 .into_iter()
441 .map(|part| {
442 let f = &f;
443 s.spawn(move || part.into_iter().map(f).collect::<Vec<R>>())
444 })
445 .collect();
446 handles.into_iter().flat_map(|h| h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect()
447 })
448}
449
450fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
452 if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
453 return Ok(Some(h));
454 }
455 store.resolve_name(name, END).map_err(|e| e.to_string())
456}
457
458fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
460 if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
461 return Ok(Some(h));
462 }
463 store.resolve_class(name, END).map_err(|e| e.to_string())
464}
465
466fn function_at<S: Store + ?Sized>(
468 reads: &mut Reads,
469 store: &S,
470 sim: &Simulation,
471 name: &str,
472 at: &str,
473) -> Result<Option<String>, String> {
474 let stored = reads.binding(store, name, at)?;
475 let simmed = Simulation::latest(&sim.functions, name, at);
476 Ok(match (stored, simmed) {
477 (Some(b), Some((t, h))) if t >= b.tx => Some(h),
478 (Some(b), _) => Some(b.fn_hash),
479 (None, Some((_, h))) => Some(h),
480 (None, None) => None,
481 })
482}
483
484fn latest_bind<S: Store + ?Sized>(
486 store: &S,
487 sim: &Simulation,
488 name: &str,
489 class: bool,
490) -> Result<Option<String>, String> {
491 let stored: Option<String> = if class {
492 store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
493 } else {
494 store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
495 };
496 let binds = if class { &sim.classes } else { &sim.functions };
497 let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
498 Ok(match (stored, planned) {
499 (Some(a), Some(b)) => Some(a.max(b)),
500 (a, b) => a.or(b),
501 })
502}
503
504fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
506 if class {
507 Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
508 } else {
509 Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
510 || sim.classes.contains_key(name))
511 }
512}
513
514fn cell_latest<S: Store + ?Sized>(
516 reads: &mut Reads,
517 store: &S,
518 sim: &Simulation,
519 id: &str,
520) -> Result<Option<(String, String)>, String> {
521 let stored = reads.latest(store, id)?;
522 let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
523 Ok(match (stored, planned) {
524 (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
525 (a, b) => a.or(b),
526 })
527}
528
529fn cell_version_asof<S: Store + ?Sized>(
531 store: &S,
532 sim: &Simulation,
533 id: &str,
534 at: &str,
535) -> Result<Option<String>, String> {
536 let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
537 let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
538 let latest = match (stored, planned) {
539 (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
540 (a, b) => a.or(b),
541 };
542 Ok(latest.map(|x| x.1))
543}
544
545fn function_bytes_of<S: Store + ?Sized>(
547 reads: &mut Reads,
548 store: &S,
549 sim: &Simulation,
550 hash: &str,
551) -> Result<Option<Vec<u8>>, String> {
552 if let Some(b) = sim.descriptors.get(hash) {
553 return Ok(Some(b.clone()));
554 }
555 Ok(reads.descriptor(store, hash))
556}
557
558fn suspended<S: Store + ?Sized>(reads: &mut Reads, store: &S, host: &str, line: usize) -> Result<(), PublishError> {
560 match reads.namespace(store, host) {
561 Ok(Some(n)) if n.suspended_at.is_some() => {
562 Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
563 }
564 Ok(_) => Ok(()),
565 Err(e) => Err(PublishError::at(line, e)),
566 }
567}
568
569struct Bind {
571 line: usize,
572 name: String,
573}
574
575#[derive(Default)]
577struct Written {
578 namespaces: BTreeSet<String>,
579 starts: BTreeSet<String>,
580}
581
582impl Written {
583 fn bound(&mut self, name: &str, start: &str) {
584 self.namespaces.insert(namespace_of(name).to_string());
585 self.starts.insert(start.to_string());
586 }
587}
588
589struct Ctx<'a> {
591 host: &'a str,
592 owned: &'a [String],
593 key_from: &'a str,
595 now: &'a str,
597 base: &'a Path,
599}
600
601enum Op {
603 Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
605 Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
607 Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
609 Cell { tx: String, id: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
611 Catalog { entry: datumesh_store::CatalogEntry, dictionaries: Vec<(String, Vec<datumesh_store::KeyValue>)> },
613}
614
615fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
617 let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
618 let mut out = Vec::new();
619 for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
620 let n = i + 1;
621 let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
622 if !line.trim().is_empty() {
623 out.push((n, line));
624 }
625 }
626 Ok(out)
627}
628
629fn index_cell_values<S: Store + ?Sized>(store: &S, ops: &[Op]) -> Result<(), String> {
633 let mut seen = BTreeSet::new();
634 for op in ops {
635 if let Op::Cell { receipt, .. } = op {
636 if let Ok(r) = datumesh_core::Receipt::from_json(receipt) {
637 cell_string_values(&r, &mut seen);
638 }
639 }
640 }
641 store_cell_values(store, seen)
642}
643
644pub(crate) fn cell_string_values(r: &datumesh_core::Receipt, out: &mut BTreeSet<(String, String, String)>) {
646 let cell = r.cell();
647 for (key, arg) in &cell.args {
648 if let datumesh_core::Arg::Value(datumesh_core::Scalar::String(v)) = arg {
649 if !v.is_empty() && v.len() <= 1024 {
650 out.insert((cell.function_name(), key.clone(), v.clone()));
651 }
652 }
653 }
654}
655
656pub(crate) fn store_cell_values<S: Store + ?Sized>(
658 store: &S,
659 seen: BTreeSet<(String, String, String)>,
660) -> Result<(), String> {
661 for (owner, key, value) in seen {
662 if crate::find::has_value(store, &owner, &key, &value) {
663 continue;
664 }
665 let kv = datumesh_store::KeyValue { value, label: String::new(), aliases: Vec::new() };
666 store.upsert_key_values(&owner, &key, std::slice::from_ref(&kv)).map_err(|e| e.to_string())?;
667 }
668 Ok(())
669}
670
671fn is_cell_line(v: &Value) -> bool {
673 v.get("class").and_then(Value::as_str).is_none()
674 && v.get("descriptor").is_none()
675 && v.get("function").and_then(Value::as_str).is_none()
676 && v.get("formula").and_then(Value::as_str).is_none()
677 && v.get("cell").is_some()
678}
679
680struct Pending {
682 line: usize,
683 ns: String,
684 cell: Cell,
685 value: Scalar,
686 valid: Args,
687 tx: String,
688 asof: String,
689 issued: String,
690 has_tx: bool,
691 licence: String,
692 summary: String,
693 blob: Option<Vec<u8>>,
694}
695
696impl Pending {
697 fn sign(self, key: &SigningKey, host: &str) -> Result<Signed, PublishError> {
699 let line = self.line;
700 let err = move |d: String| PublishError::at(line, d);
701 let r = ReceiptBuilder::new()
702 .cell(self.cell.clone())
703 .value(self.value)
704 .valid(self.valid)
705 .asof(self.asof.as_str())
706 .tx(self.tx.as_str())
707 .issued(self.issued.as_str())
708 .publisher(self.ns.as_str())
709 .signer_host(host)
710 .licence(self.licence.as_str())
711 .summary(self.summary.as_str())
712 .sign(key)
713 .map_err(|e| err(e.to_string()))?;
714 let receipt = receipt_json(&r);
715 let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
716 let id = self.cell.id();
717 Ok(Signed { line, has_tx: self.has_tx, tx: self.tx, id, hash, receipt, blob: self.blob })
718 }
719}
720
721struct Signed {
723 line: usize,
724 has_tx: bool,
725 tx: String,
726 id: String,
727 hash: String,
728 receipt: Vec<u8>,
729 blob: Option<Vec<u8>>,
730}
731
732struct Planner<'a, S: Store + ?Sized> {
734 store: &'a S,
735 key: &'a SigningKey,
736 ctx: Ctx<'a>,
737 backfill: bool,
738 sim: Simulation,
739 reads: Reads,
740 ops: Vec<Op>,
741 binds: Vec<Bind>,
742 tests: Vec<(usize, TestSpec)>,
744 formulas: Vec<(usize, String, String)>,
746}
747
748impl<S: Store + ?Sized> Planner<'_, S> {
749 fn callers(&self) -> Callers<'_, S> {
750 Callers { store: self.store, sim: Some(&self.sim) }
751 }
752
753 fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
756 let mut cells: Vec<(usize, Value)> = Vec::new();
757 for (n, text) in lines {
758 let n = *n;
759 match json::parse(text.as_bytes()).map_err(|e| PublishError::at(n, e.to_string())) {
760 Ok(v) if is_cell_line(&v) => {
761 cells.push((n, v));
762 if cells.len() == CHUNK {
763 self.plan_cells(std::mem::take(&mut cells))?;
764 }
765 }
766 other => {
767 self.plan_cells(std::mem::take(&mut cells))?;
770 self.plan_line(n, &other?)?;
771 }
772 }
773 }
774 self.plan_cells(cells)
775 }
776
777 fn plan_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
779 let err = |d: String| PublishError::at(n, d);
780 if let Some(name) = v.get("class").and_then(Value::as_str) {
781 self.class_line(n, name, v)?;
782 } else if let Some(d) = v.get("descriptor") {
783 self.descriptor_line(n, d)?;
784 } else if let Some(name) = v.get("function").and_then(Value::as_str) {
785 let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
786 let start = bound_of(v, n, self.ctx.now)?;
787 let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
788 self.function(n, name, start, fd.hash(), fd.canonical())?;
789 } else if v.get("catalog").is_some() {
790 self.catalog_line(n, v)?;
791 } else if let Some(name) = v.get("formula").and_then(Value::as_str) {
792 let source =
793 v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
794 let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
795 let start = bound_of(v, n, self.ctx.now)?;
796 let def = datumesh_formula::definition::define_function(
797 name,
798 source,
799 sig,
800 &Default::default(),
801 &self.callers(),
802 &start,
803 )
804 .map_err(|e| err(format!("define {name}: {e}")))?;
805 let hash = def.fn_hash.clone();
806 self.function(n, name, start, def.fn_hash, def.bytes)?;
807 self.formulas.push((n, name.to_string(), hash));
808 } else if v.get("test").is_some() {
809 let spec = testrun::parse_test(v).map_err(err)?;
810 self.tests.push((n, spec));
811 } else {
812 return Err(err("neither a function nor a cell".into()));
813 }
814 Ok(())
815 }
816
817 fn catalog_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
821 let err = |d: String| PublishError::at(n, d);
822 let d = crate::catalog::parse(v, "catalog").map_err(err)?;
823 let host = host_of(&d.name);
824 if host != self.ctx.host && !self.ctx.owned.iter().any(|o| o == host) {
825 return Err(err(format!("{} is not under a host this mesh owns", d.name)));
826 }
827 let hash = match d.kind {
828 datumesh_store::CatalogKind::Class => current_class(self.store, &self.sim, &d.name).map_err(err)?,
829 _ => current_function(self.store, &self.sim, &d.name).map_err(err)?,
830 };
831 if hash.is_none() {
832 return Err(err(format!("{} is bound nowhere", d.name)));
833 }
834 let entry = d.entry(hash, datumesh_store::CatalogSource::Publisher, Some(self.key.key_id().to_string()));
835 self.ops.push(Op::Catalog { entry, dictionaries: d.dictionary });
836 Ok(())
837 }
838
839 fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
841 let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
842 if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
843 return Err(PublishError::at(
844 n,
845 "a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
846 ));
847 }
848 let (hash, bytes) = (fd.hash(), fd.canonical());
849 self.sim.descriptors.insert(hash.clone(), bytes.clone());
850 self.ops.push(Op::Descriptor { hash, bytes, counted: true });
851 Ok(())
852 }
853
854 fn function(
856 &mut self,
857 n: usize,
858 name: &str,
859 start: String,
860 hash: String,
861 bytes: Vec<u8>,
862 ) -> Result<(), PublishError> {
863 self.sim.descriptors.insert(hash.clone(), bytes.clone());
864 let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
865 let bind = current.as_deref() != Some(hash.as_str());
866 if bind {
867 self.check_bind(n, name, &start, false)?;
868 self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
869 }
870 self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
871 Ok(())
872 }
873
874 fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
875 let err = |d: String| PublishError::at(n, d);
876 let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
877 let start = bound_of(v, n, self.ctx.now)?;
878 let def = datumesh_formula::definition::define_class(
879 name,
880 &descriptor.to_jcs(),
881 definer(name, self.ctx.host, self.ctx.owned),
882 &self.callers(),
883 &start,
884 )
885 .map_err(|e| err(format!("define {name}: {e}")))?;
886 suspended(&mut self.reads, self.store, host_of(&def.descriptor.steward.namespace), n)?;
890 let hash = def.class_hash.clone();
891 let methods: Vec<(String, String)> =
892 def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
893 let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
894 if bind {
895 self.check_bind(n, name, &start, true)?;
896 for (mname, _) in &methods {
898 self.check_bind(n, mname, &start, false)?;
899 }
900 self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
901 for (mname, fh) in &methods {
902 self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
903 }
904 }
905 self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
906 self.ops.push(Op::Class {
907 name: name.to_string(),
908 start,
909 hash,
910 bytes: def.bytes,
911 bind,
912 methods: methods.into_iter().map(|(m, _)| m).collect(),
913 });
914 Ok(())
915 }
916
917 fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
919 if !self.backfill && start < self.ctx.now {
920 return Err(PublishError::typed(
921 line,
922 PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
923 ));
924 }
925 let err = |d: String| PublishError::at(line, d);
926 if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
927 let kind = if class { "function" } else { "class" };
928 return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
929 }
930 suspended(&mut self.reads, self.store, host_of(name), line)?;
931 if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
932 if start <= latest.as_str() {
933 return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
934 }
935 }
936 self.binds.push(Bind { line, name: name.into() });
937 Ok(())
938 }
939
940 fn plan_cells(&mut self, lines: Vec<(usize, Value)>) -> Result<(), PublishError> {
945 if lines.is_empty() {
946 return Ok(());
947 }
948 let mut pending = Vec::with_capacity(lines.len());
949 let mut failed = None;
950 for (n, v) in &lines {
951 match self.prepare_cell(*n, v) {
952 Ok(p) => pending.push(p),
953 Err(e) => {
954 failed = Some(e);
955 break;
956 }
957 }
958 }
959 let (key, host) = (self.key, self.ctx.host);
960 let signed = parallel_owned(pending, |p| p.sign(key, host));
961 let ids: Vec<&str> = signed.iter().flatten().map(|s| s.id.as_str()).collect();
962 self.reads.prefetch_latest(self.store, &ids);
963 let hashes: Vec<String> = signed
964 .iter()
965 .flatten()
966 .filter(|s| !s.has_tx)
967 .filter_map(|s| self.reads.latest_of.get(&s.id).cloned().flatten())
968 .map(|(_, h)| h)
969 .collect();
970 let hashes: Vec<&str> = hashes.iter().map(String::as_str).collect();
971 self.reads.prefetch_receipts(self.store, &hashes);
972 for s in signed {
973 self.check_cell(s?)?;
974 }
975 match failed {
976 Some(e) => Err(e),
977 None => Ok(()),
978 }
979 }
980
981 fn prepare_cell(&mut self, n: usize, v: &Value) -> Result<Pending, PublishError> {
984 let err = |d: String| PublishError::at(n, d);
985 let cell_v = v.get("cell").ok_or_else(|| err("a cell without fn".into()))?;
986 let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
987 let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
988 let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
989 let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
990 let (licence, summary) = (required("licence")?, required("summary")?);
991 let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
993 let has_tx = field("tx").is_some();
994 let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
995 if tx > asof {
996 return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
997 }
998 let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
999 if asof > issued {
1000 return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
1001 }
1002 let fn_hash = function_at(&mut self.reads, self.store, &self.sim, name, &tx)
1003 .map_err(err)?
1004 .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
1005 let d = function_bytes_of(&mut self.reads, self.store, &self.sim, &fn_hash)
1006 .map_err(err)?
1007 .ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
1008 let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
1009 let mut args = Args::new();
1010 for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
1011 args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
1012 }
1013 let valid: Args = args
1014 .iter()
1015 .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
1016 .map(|(k, a)| (k.clone(), a.clone()))
1017 .collect();
1018 let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
1019 let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
1020 Some(file) => {
1021 let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
1022 let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
1023 let sha = sha256_hex(&bytes);
1024 let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
1025 .map_err(|e| err(e.to_string()))?;
1026 (scalar, Some(bytes))
1027 }
1028 None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
1029 };
1030 let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
1031 Ok(Pending {
1032 line: n,
1033 ns: ns.to_string(),
1034 cell,
1035 value,
1036 valid,
1037 tx,
1038 asof,
1039 issued,
1040 has_tx,
1041 licence,
1042 summary,
1043 blob,
1044 })
1045 }
1046
1047 fn check_cell(&mut self, s: Signed) -> Result<(), PublishError> {
1050 let line = s.line;
1051 let err = |d: String| PublishError::at(line, d);
1052 if !s.has_tx {
1057 if let Some((_, latest)) = cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1058 if let Some(old) = self.reads.receipt(self.store, &latest) {
1059 let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
1060 let new = json::parse(&s.receipt).map_err(|e| err(e.to_string()))?;
1061 if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
1062 return Ok(());
1063 }
1064 }
1065 }
1066 }
1067 let version = match cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1068 Some((latest_tx, _)) if latest_tx.as_str() >= s.tx.as_str() => {
1069 if cell_version_asof(self.store, &self.sim, &s.id, &s.tx).map_err(err)?.as_deref()
1070 != Some(s.hash.as_str())
1071 {
1072 return Err(err(format!("{} has a version at or after {} that is not this receipt", s.id, s.tx)));
1073 }
1074 false
1075 }
1076 _ => {
1077 self.sim.versions.entry(s.id.clone()).or_default().push((s.tx.clone(), s.hash.clone()));
1078 true
1079 }
1080 };
1081 self.ops.push(Op::Cell { tx: s.tx, id: s.id, receipt: s.receipt, blob: s.blob, version });
1082 Ok(())
1083 }
1084}
1085
1086fn stored_value(signature: &Value) -> Value {
1088 Value::Object(vec![
1089 ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
1090 ("inputs".into(), Value::Array(vec![])),
1091 ("signature".into(), signature.clone()),
1092 ])
1093}
1094
1095fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
1098 for b in binds {
1099 let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
1100 if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
1101 return Err(refuse(format!("{} has a binding", b.name)));
1102 }
1103 let ns = namespace_of(&b.name);
1104 if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
1105 return Err(refuse(format!("{ns} has bindings")));
1106 }
1107 }
1108 Ok(())
1109}
1110
1111fn append_log(log: &Path, written: &Written) -> Result<(), String> {
1113 let line = Value::Object(vec![
1114 ("at".into(), s(&SystemClock.now())),
1115 ("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
1116 ("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
1117 ])
1118 .to_jcs();
1119 let mut opts = std::fs::OpenOptions::new();
1120 opts.create(true).append(true);
1121 #[cfg(unix)]
1122 {
1123 use std::os::unix::fs::OpenOptionsExt;
1124 opts.mode(0o600);
1125 }
1126 let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
1127 let mut bytes = line;
1128 bytes.push(b'\n');
1129 f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
1130}
1131
1132fn write<S: Store + ?Sized>(
1136 store: &S,
1137 ops: &[Op],
1138 cells: impl Fn(&S, &[Op], &mut Published) -> Result<(), String>,
1139) -> Result<(Published, Written), PublishError> {
1140 let mut out = Published::default();
1141 let mut written = Written::default();
1142 let err = |d: String| PublishError::at(0, d);
1143 let mut i = 0;
1144 while i < ops.len() {
1145 match &ops[i] {
1146 Op::Descriptor { hash, bytes, counted } => {
1147 let existed = store.descriptor(hash).is_some();
1148 store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1149 if *counted && !existed {
1150 out.descriptors += 1;
1151 }
1152 }
1153 Op::Function { name, start, hash, bytes, bind } => {
1154 store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1155 if *bind {
1156 store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1157 out.functions += 1;
1158 written.bound(name, start);
1159 }
1160 }
1161 Op::Class { name, start, hash, bytes, bind, methods } => {
1162 store.put_class(bytes).map_err(|e| err(e.to_string()))?;
1163 if *bind {
1164 store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1165 out.classes += 1;
1166 written.bound(name, start);
1167 for m in methods {
1168 written.bound(m, start);
1169 }
1170 }
1171 }
1172 Op::Catalog { entry, dictionaries } => {
1173 crate::catalog::store_publisher(store, entry, dictionaries).map_err(err)?;
1174 }
1175 Op::Cell { .. } => {
1176 let mut j = i;
1177 while j < ops.len() && j - i < CHUNK && matches!(ops[j], Op::Cell { .. }) {
1178 j += 1;
1179 }
1180 cells(store, &ops[i..j], &mut out).map_err(err)?;
1181 i = j;
1182 continue;
1183 }
1184 }
1185 i += 1;
1186 }
1187 Ok((out, written))
1188}
1189
1190fn write_cells_one_lane<S: Store + ?Sized>(store: &S, group: &[Op], out: &mut Published) -> Result<(), String> {
1194 let items = put_blobs_and_items(store, group, out)?;
1195 let failed = AtomicBool::new(false);
1196 let ks: Vec<usize> = (0..items.len()).collect();
1197 if let Some((_, detail)) = put_lane(store, &items, &ks, &failed) {
1198 return Err(detail);
1199 }
1200 count_cells(group, &items, out);
1201 Ok(())
1202}
1203
1204fn write_cells<S: Store + Sync + ?Sized>(
1210 store: &S,
1211 group: &[Op],
1212 lanes: usize,
1213 out: &mut Published,
1214) -> Result<(), String> {
1215 if lanes <= 1 {
1216 return write_cells_one_lane(store, group, out);
1217 }
1218 let items = put_blobs_and_items(store, group, out)?;
1219 let mut by_lane: Vec<Vec<usize>> = vec![Vec::new(); lanes];
1220 for (k, op) in group.iter().enumerate() {
1221 if let Op::Cell { id, .. } = op {
1222 by_lane[lane_of(id, lanes)].push(k);
1223 }
1224 }
1225 let failed = AtomicBool::new(false);
1226 let failures: Vec<(usize, String)> = std::thread::scope(|scope| {
1227 let handles: Vec<_> = by_lane
1228 .iter()
1229 .map(|ks| {
1230 let (items, failed) = (&items, &failed);
1231 scope.spawn(move || put_lane(store, items, ks, failed))
1232 })
1233 .collect();
1234 handles.into_iter().filter_map(|h| h.join().unwrap_or_else(|p| std::panic::resume_unwind(p))).collect()
1235 });
1236 if let Some((_, detail)) = failures.into_iter().min_by_key(|(k, _)| *k) {
1237 return Err(detail);
1238 }
1239 count_cells(group, &items, out);
1240 Ok(())
1241}
1242
1243fn put_blobs_and_items<'a, S: Store + ?Sized>(
1245 store: &S,
1246 group: &'a [Op],
1247 out: &mut Published,
1248) -> Result<Vec<CellItem<'a>>, String> {
1249 for op in group {
1250 if let Op::Cell { blob: Some(bytes), .. } = op {
1251 store.put_blob(bytes).map_err(|e| e.to_string())?;
1252 out.blobs += 1;
1253 out.blob_bytes += bytes.len() as u64;
1254 }
1255 }
1256 Ok(group
1257 .iter()
1258 .filter_map(|op| match op {
1259 Op::Cell { tx, id, receipt, version, .. } => {
1260 Some((receipt.as_slice(), if *version { Some((id.as_str(), tx.as_str())) } else { None }))
1261 }
1262 _ => None,
1263 })
1264 .collect())
1265}
1266
1267fn count_cells(group: &[Op], items: &[CellItem<'_>], out: &mut Published) {
1269 out.cells += group.len();
1270 out.receipts += group.len();
1271 out.versions += items.iter().filter(|(_, v)| v.is_some()).count();
1272}
1273
1274fn lane_of(id: &str, lanes: usize) -> usize {
1276 id.bytes().next().and_then(|b| (b as char).to_digit(16)).unwrap_or(0) as usize % lanes
1277}
1278
1279type CellItem<'a> = (&'a [u8], Option<(&'a str, &'a str)>);
1281
1282fn put_lane<S: Store + ?Sized>(
1286 store: &S,
1287 items: &[CellItem<'_>],
1288 ks: &[usize],
1289 failed: &AtomicBool,
1290) -> Option<(usize, String)> {
1291 for call in ks.chunks(CHUNK) {
1292 if failed.load(Ordering::SeqCst) {
1293 return None;
1294 }
1295 let writes: Vec<CellWrite<'_>> = call.iter().map(|&k| CellWrite::new(items[k].0, items[k].1)).collect();
1296 if let Err(e) = store.put_cells(&writes) {
1297 failed.store(true, Ordering::SeqCst);
1298 return Some((call[0], e.to_string()));
1299 }
1300 }
1301 None
1302}
1303
1304pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
1306 let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
1307 if let Ok(Value::Object(m)) = json::parse(r.payload()) {
1308 members.extend(m);
1309 }
1310 Value::Object(members).to_jcs()
1311}
1312
1313pub fn publish_stored<S: Store + ?Sized>(
1318 store: &S,
1319 key: &SigningKey,
1320 host: &str,
1321 path: &Path,
1322) -> Result<Published, PublishError> {
1323 publish_stored_in(store, key, host, &[host.to_string()], path)
1324}
1325
1326pub fn publish_stored_in<S: Store + ?Sized>(
1330 store: &S,
1331 key: &SigningKey,
1332 host: &str,
1333 owned: &[String],
1334 path: &Path,
1335) -> Result<Published, PublishError> {
1336 publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
1337}
1338
1339pub fn publish_stored_with<S: Store + ?Sized>(
1346 store: &S,
1347 key: &SigningKey,
1348 host: &str,
1349 owned: &[String],
1350 path: &Path,
1351 opts: &PublishOptions,
1352) -> Result<Published, PublishError> {
1353 publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1354 write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1355 })
1356}
1357
1358pub fn publish_stored_checked<S: Store + ?Sized>(
1365 store: &S,
1366 key: &SigningKey,
1367 host: &str,
1368 owned: &[String],
1369 path: &Path,
1370 opts: &PublishOptions,
1371) -> Result<Published, PublishError> {
1372 publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1373 write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1374 })
1375}
1376
1377pub fn publish_stored_lanes<S: Store + Sync + ?Sized>(
1382 store: &S,
1383 key: &SigningKey,
1384 host: &str,
1385 owned: &[String],
1386 path: &Path,
1387 opts: &PublishOptions,
1388) -> Result<Published, PublishError> {
1389 let lanes = publish_lanes()?;
1390 publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1391 write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1392 })
1393}
1394
1395fn overlay_of(ops: &[Op]) -> Result<Overlay, PublishError> {
1397 let mut overlay = Overlay::default();
1398 for op in ops {
1399 match op {
1400 Op::Function { name, start, hash, bytes, bind } => {
1401 overlay.descriptors.insert(hash.clone(), bytes.clone());
1402 if *bind {
1403 overlay.functions.entry(name.clone()).or_default().push((start.clone(), hash.clone()));
1404 }
1405 }
1406 Op::Descriptor { hash, bytes, .. } => {
1407 overlay.descriptors.insert(hash.clone(), bytes.clone());
1408 }
1409 Op::Cell { receipt, .. } => {
1410 overlay.cell(receipt).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1411 }
1412 Op::Class { .. } | Op::Catalog { .. } => {}
1413 }
1414 }
1415 Ok(overlay)
1416}
1417
1418#[allow(clippy::too_many_arguments)]
1423pub(crate) fn check_tests<S: Store + ?Sized>(
1424 store: &S,
1425 overlay: &Overlay,
1426 tests: &[(usize, TestSpec)],
1427 formulas: &[(usize, String, String)],
1428 host: &str,
1429 now: &str,
1430 checked: bool,
1431) -> Result<Vec<RowWrite>, PublishError> {
1432 if formulas.is_empty() {
1433 return Ok(Vec::new());
1434 }
1435 let mut file_tests: Vec<(usize, TestSpec, String)> = Vec::new();
1436 for (n, spec) in tests {
1437 let at = spec.asof.clone().unwrap_or_else(|| now.to_string());
1438 match testrun::binding_of(store, overlay, &spec.function, &at) {
1439 Ok(Some((hash, _))) => file_tests.push((*n, spec.clone(), hash)),
1440 Ok(None) => {
1441 return Err(PublishError::at(
1442 *n,
1443 format!("a test of {}: no function of that name is bound", spec.function),
1444 ))
1445 }
1446 Err(e) => return Err(PublishError::at(*n, e)),
1447 }
1448 }
1449 let mut rows = Vec::new();
1450 for (line, name, hash) in formulas {
1451 let own: Vec<(usize, TestSpec)> =
1452 file_tests.iter().filter(|t| &t.2 == hash).map(|t| (t.0, t.1.clone())).collect();
1453 let held = testrun::held_tests(store, hash).map_err(|e| PublishError::at(*line, e))?;
1454 if !checked && own.is_empty() && held.is_empty() {
1455 continue;
1456 }
1457 let mut specs: Vec<(Option<usize>, TestSpec)> = own.into_iter().map(|(n, s)| (Some(n), s)).collect();
1458 for (_, spec) in held {
1459 if !specs.iter().any(|(_, s)| s.case == spec.case) {
1460 specs.push((None, spec));
1461 }
1462 }
1463 let ran: Vec<Ran> = specs.iter().map(|(_, s)| run_test(store, overlay, now, s)).collect();
1464 let mut run = testrun::Run::new(store, overlay, now);
1465 let built = run.built(name, hash).map_err(|e| PublishError::at(*line, format!("{name}: {}", e.0)))?;
1466 let Some(compiled) = built.compiled.as_ref() else { continue };
1467 if checked {
1468 let mut failing: Vec<(usize, String)> = Vec::new();
1469 for ((n, spec), r) in specs.iter().zip(&ran) {
1470 if let Some(n) = n {
1471 let (result, detail) = testrun::result_of(spec, r);
1472 if result != "pass" {
1473 failing.push((*n, format!("{name}: test {} {detail}", spec.case)));
1474 }
1475 }
1476 }
1477 failing.sort_by_key(|f| f.0);
1478 if let Some((n, detail)) = failing.into_iter().next() {
1479 return Err(PublishError::typed(n, PublishErrorKind::TestsFailing(detail)));
1480 }
1481 let refs: Vec<&Ran> = ran.iter().collect();
1482 let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1483 if !cov.uncovered.is_empty() || !cov.unexercised.is_empty() {
1484 let mut text = format!("{name} is missing tests:");
1485 if !cov.uncovered.is_empty() {
1486 let cases: Vec<&str> = cov.uncovered.iter().map(|c| c.0.as_str()).collect();
1487 text += &format!(" uncovered cases {};", cases.join(", "));
1488 }
1489 if !cov.unexercised.is_empty() {
1490 text += &format!(" unexercised branches {}", cov.unexercised.join(", "));
1491 }
1492 return Err(PublishError::typed(
1493 *line,
1494 PublishErrorKind::TestsMissing(text.trim_end_matches(';').to_string()),
1495 ));
1496 }
1497 }
1498 let refs: Vec<&Ran> = ran.iter().collect();
1499 let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1500 for ((_, spec), r) in specs.iter().zip(&ran) {
1501 let (result, detail) = testrun::result_of(spec, r);
1502 rows.push(RowWrite {
1503 fn_hash: hash.clone(),
1504 case: spec.case.clone(),
1505 jcs: spec.jcs.clone(),
1506 signed_by: host.to_string(),
1507 result,
1508 detail,
1509 branches_total: cov.branches,
1510 branches_covered: cov.covered,
1511 });
1512 }
1513 }
1514 Ok(rows)
1515}
1516
1517pub fn publish_stored_checked_lanes<S: Store + Sync + ?Sized>(
1520 store: &S,
1521 key: &SigningKey,
1522 host: &str,
1523 owned: &[String],
1524 path: &Path,
1525 opts: &PublishOptions,
1526) -> Result<Published, PublishError> {
1527 let lanes = publish_lanes()?;
1528 publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1529 write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1530 })
1531}
1532
1533#[allow(clippy::too_many_arguments)]
1535fn publish_planned<S: Store + ?Sized>(
1536 store: &S,
1537 key: &SigningKey,
1538 host: &str,
1539 owned: &[String],
1540 path: &Path,
1541 opts: &PublishOptions,
1542 checked: bool,
1543 write_ops: impl FnOnce(&S, &[Op]) -> Result<(Published, Written), PublishError>,
1544) -> Result<Published, PublishError> {
1545 let lines = read_lines(path)?;
1546 let key_from = store
1548 .namespace(host)
1549 .ok()
1550 .flatten()
1551 .and_then(|n| json::parse(&n.keydoc).ok())
1552 .and_then(|d| match d.get("keys") {
1553 Some(Value::Array(ks)) => ks
1554 .iter()
1555 .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
1556 .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
1557 _ => None,
1558 })
1559 .ok_or_else(|| {
1560 PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
1561 })?;
1562 let now = opts.clock.now();
1563 let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
1564 let mut planner = Planner {
1565 store,
1566 key,
1567 ctx,
1568 backfill: opts.backfill.is_some(),
1569 sim: Simulation::default(),
1570 reads: Reads::default(),
1571 ops: Vec::new(),
1572 binds: Vec::new(),
1573 tests: Vec::new(),
1574 formulas: Vec::new(),
1575 };
1576 planner.run(&lines)?;
1577 let Planner { ops, binds, tests, formulas, .. } = planner;
1578 if opts.backfill.is_some() {
1579 backfill_check(store, &binds)?;
1580 }
1581 let rows = check_tests(store, &overlay_of(&ops)?, &tests, &formulas, host, &now, checked)?;
1584 let (out, written) = write_ops(store, &ops)?;
1585 index_cell_values(store, &ops).map_err(|e| PublishError::at(0, format!("catalogue: {e}")))?;
1586 testrun::apply_rows(store, &rows, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1587 let rebound: Vec<String> = ops
1588 .iter()
1589 .filter_map(|op| match op {
1590 Op::Function { name, bind: true, .. } => Some(name.clone()),
1591 _ => None,
1592 })
1593 .collect();
1594 testrun::rerun_readers(store, &rebound, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1595 if let Some(log) = &opts.backfill {
1596 if out.functions + out.classes + out.versions > 0 {
1597 append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
1598 }
1599 }
1600 Ok(out)
1601}