1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
62use std::io::{BufRead, Write};
63use std::path::{Path, PathBuf};
64use std::sync::atomic::{AtomicBool, Ordering};
65use std::sync::Arc;
66
67use datumesh_core::json::{self, Value};
68use datumesh_core::{
69 sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey,
70 Source, Type,
71};
72use datumesh_engine::{Clock, SystemClock};
73use datumesh_formula::definition::{Callee, Callees, ClassCallee};
74use datumesh_store::{Binding, CellWrite, Namespace, Store};
75
76use crate::testrun::{self, run_test, Overlay, Ran, RowWrite, TestSpec};
77
78use crate::config::ConfigError;
79
80const END: &str = "9999-12-31T23:59:59.999999Z";
82
83const CHUNK: usize = 1_000;
85
86const LANES_VAR: &str = "DATUMESH_PUBLISH_LANES";
88
89fn publish_lanes() -> Result<usize, PublishError> {
92 let bad = |detail: String| PublishError::at(0, ConfigError::new(LANES_VAR, detail).to_string());
93 match std::env::var(LANES_VAR) {
94 Err(std::env::VarError::NotPresent) => Ok(std::thread::available_parallelism().map_or(1, |n| n.get()).min(4)),
95 Ok(v) => match v.parse::<usize>() {
96 Ok(n) if n > 0 => Ok(n),
97 _ => Err(bad(format!("must be a positive integer, not {v:?}"))),
98 },
99 Err(e) => Err(bad(e.to_string())),
100 }
101}
102
103#[non_exhaustive]
105#[derive(Clone, Debug, Default, PartialEq, Eq)]
106pub struct Published {
107 pub functions: usize,
109 pub classes: usize,
111 pub cells: usize,
113 pub receipts: usize,
115 pub versions: usize,
117 pub blobs: usize,
119 pub blob_bytes: u64,
121 pub descriptors: u64,
123}
124
125#[derive(Clone)]
127pub struct PublishOptions {
128 pub clock: Arc<dyn Clock + Send + Sync>,
131 pub backfill: Option<PathBuf>,
135}
136
137impl PublishOptions {
138 pub fn system() -> PublishOptions {
140 PublishOptions { clock: Arc::new(SystemClock), backfill: None }
141 }
142}
143
144#[non_exhaustive]
146#[derive(Clone, Debug, PartialEq, Eq)]
147pub enum PublishErrorKind {
148 Line,
150 StartBeforeNow {
152 name: String,
154 start: String,
156 now: String,
158 },
159 Backfill(String),
161 OwnedVectorName(String),
163 TestsMissing(String),
166 TestsFailing(String),
169}
170
171impl std::fmt::Display for PublishErrorKind {
172 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
173 match self {
174 PublishErrorKind::Line => f.write_str("refused"),
175 PublishErrorKind::StartBeforeNow { name, start, now } => {
176 write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
177 }
178 PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
179 PublishErrorKind::OwnedVectorName(name) => {
180 write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
181 }
182 PublishErrorKind::TestsMissing(text) => write!(f, "E_TESTS_MISSING: {text}"),
183 PublishErrorKind::TestsFailing(text) => write!(f, "E_TESTS_FAILING: {text}"),
184 }
185 }
186}
187
188#[derive(Clone, Debug, PartialEq, Eq)]
190pub struct PublishError {
191 pub line: usize,
193 pub detail: String,
195 pub kind: PublishErrorKind,
197}
198
199impl PublishError {
200 fn at(line: usize, detail: impl Into<String>) -> PublishError {
201 PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
202 }
203
204 pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
205 PublishError { line, detail: kind.to_string(), kind }
206 }
207}
208
209impl std::fmt::Display for PublishError {
210 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
211 write!(f, "line {}: {}", self.line, self.detail)
212 }
213}
214
215impl std::error::Error for PublishError {}
216
217fn s(v: &str) -> Value {
218 Value::String(v.to_string())
219}
220
221fn namespace_of(name: &str) -> &str {
223 name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
224}
225
226fn host_of(name: &str) -> &str {
228 name.split('/').next().unwrap_or("")
229}
230
231fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
234 let host_of_name = name.split('/').next().unwrap_or("");
235 if owned.iter().any(|o| o == host_of_name) {
236 namespace_of(name)
237 } else {
238 host
239 }
240}
241
242fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
244 match v.get("bound") {
245 None => Ok(now.to_string()),
246 Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
247 }
248}
249
250#[derive(Default)]
253struct Simulation {
254 functions: BTreeMap<String, Vec<(String, String)>>,
256 classes: BTreeMap<String, Vec<(String, String)>>,
258 descriptors: BTreeMap<String, Vec<u8>>,
260 class_bytes: BTreeMap<String, Vec<u8>>,
262 versions: BTreeMap<String, Vec<(String, String)>>,
264}
265
266impl Simulation {
267 fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
269 binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
270 }
271}
272
273struct Callers<'a, S: Store + ?Sized> {
275 store: &'a S,
276 sim: Option<&'a Simulation>,
277}
278
279impl<S: Store + ?Sized> Callers<'_, S> {
280 fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
281 if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
282 return Ok(Some(b.clone()));
283 }
284 Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
285 }
286
287 fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
288 if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
289 return Ok(Some(b.clone()));
290 }
291 self.store.class(hash).map_err(|e| e.to_string())
292 }
293}
294
295impl<S: Store + ?Sized> Callees for Callers<'_, S> {
296 fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
297 let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
298 let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
299 let (fn_hash, start) = match (stored, simmed) {
300 (Some(b), Some((t, h))) if t >= b.tx => (h, t),
301 (Some(b), _) => (b.fn_hash, b.tx),
302 (None, Some((t, h))) => (h, t),
303 (None, None) => return Ok(None),
304 };
305 let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
306 let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
307 Ok(Some(Callee { fn_hash, start, descriptor }))
308 }
309
310 fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
311 let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
312 let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
313 let (class_hash, start) = match (stored, simmed) {
314 (Some(b), Some((t, h))) if t >= b.tx => (h, t),
315 (Some(b), _) => (b.class_hash, b.tx),
316 (None, Some((t, h))) => (h, t),
317 (None, None) => return Ok(None),
318 };
319 let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
320 let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
321 Ok(Some(ClassCallee { class_hash, start, descriptor }))
322 }
323
324 fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
325 let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
326 ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
327 }
328
329 fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
330 let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
331 json::parse(&bytes).map(Some).map_err(|e| e.to_string())
332 }
333}
334
335#[derive(Default)]
340struct Reads {
341 bindings: HashMap<(String, String), Result<Option<Binding>, String>>,
342 descriptors: HashMap<String, Option<Vec<u8>>>,
343 namespaces: HashMap<String, Result<Option<Namespace>, String>>,
344 latest_of: HashMap<String, Option<(String, String)>>,
346 receipt_of: HashMap<String, Option<Vec<u8>>>,
348}
349
350impl Reads {
351 fn binding<S: Store + ?Sized>(&mut self, store: &S, name: &str, at: &str) -> Result<Option<Binding>, String> {
352 let key = (name.to_string(), at.to_string());
353 if let Some(r) = self.bindings.get(&key) {
354 return r.clone();
355 }
356 let r = store.resolve_binding(name, at).map_err(|e| e.to_string());
357 self.bindings.insert(key, r.clone());
358 r
359 }
360
361 fn descriptor<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
362 if let Some(b) = self.descriptors.get(hash) {
363 return b.clone();
364 }
365 let b = store.descriptor(hash).map(|b| b.into_owned());
366 self.descriptors.insert(hash.to_string(), b.clone());
367 b
368 }
369
370 fn namespace<S: Store + ?Sized>(&mut self, store: &S, host: &str) -> Result<Option<Namespace>, String> {
371 if let Some(r) = self.namespaces.get(host) {
372 return r.clone();
373 }
374 let r = store.namespace(host).map_err(|e| e.to_string());
375 self.namespaces.insert(host.to_string(), r.clone());
376 r
377 }
378
379 fn latest<S: Store + ?Sized>(&mut self, store: &S, id: &str) -> Result<Option<(String, String)>, String> {
381 if let Some(l) = self.latest_of.get(id) {
382 return Ok(l.clone());
383 }
384 store.latest(id).map(|v| v.map(|v| (v.tx, v.receipt_hash))).map_err(|e| e.to_string())
385 }
386
387 fn receipt<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
389 if let Some(b) = self.receipt_of.get(hash) {
390 return b.clone();
391 }
392 store.receipt(hash).map(|b| b.into_owned())
393 }
394
395 fn prefetch_latest<S: Store + ?Sized>(&mut self, store: &S, ids: &[&str]) {
397 let mut seen = HashSet::new();
398 let missing: Vec<&str> =
399 ids.iter().copied().filter(|id| !self.latest_of.contains_key(*id) && seen.insert(*id)).collect();
400 if missing.is_empty() {
401 return;
402 }
403 if let Ok(found) = store.latest_many(&missing) {
404 for (id, v) in missing.iter().zip(found) {
405 self.latest_of.insert(id.to_string(), v.map(|v| (v.tx, v.receipt_hash)));
406 }
407 }
408 }
409
410 fn prefetch_receipts<S: Store + ?Sized>(&mut self, store: &S, hashes: &[&str]) {
412 let mut seen = HashSet::new();
413 let missing: Vec<&str> =
414 hashes.iter().copied().filter(|h| !self.receipt_of.contains_key(*h) && seen.insert(*h)).collect();
415 if missing.is_empty() {
416 return;
417 }
418 if let Ok(found) = store.receipts_many(&missing) {
419 for (h, b) in missing.iter().zip(found) {
420 self.receipt_of.insert(h.to_string(), b);
421 }
422 }
423 }
424}
425
426fn parallel_owned<T: Send, R: Send>(items: Vec<T>, f: impl Fn(T) -> R + Sync) -> Vec<R> {
428 let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(items.len());
429 if threads <= 1 {
430 return items.into_iter().map(f).collect();
431 }
432 let size = items.len().div_ceil(threads);
433 let mut parts: Vec<Vec<T>> = Vec::with_capacity(threads);
434 let mut rest = items.into_iter();
435 loop {
436 let part: Vec<T> = rest.by_ref().take(size).collect();
437 if part.is_empty() {
438 break;
439 }
440 parts.push(part);
441 }
442 std::thread::scope(|s| {
443 let handles: Vec<_> = parts
444 .into_iter()
445 .map(|part| {
446 let f = &f;
447 s.spawn(move || part.into_iter().map(f).collect::<Vec<R>>())
448 })
449 .collect();
450 handles.into_iter().flat_map(|h| h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect()
451 })
452}
453
454fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
456 if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
457 return Ok(Some(h));
458 }
459 store.resolve_name(name, END).map_err(|e| e.to_string())
460}
461
462fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
464 if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
465 return Ok(Some(h));
466 }
467 store.resolve_class(name, END).map_err(|e| e.to_string())
468}
469
470fn function_at<S: Store + ?Sized>(
472 reads: &mut Reads,
473 store: &S,
474 sim: &Simulation,
475 name: &str,
476 at: &str,
477) -> Result<Option<String>, String> {
478 let stored = reads.binding(store, name, at)?;
479 let simmed = Simulation::latest(&sim.functions, name, at);
480 Ok(match (stored, simmed) {
481 (Some(b), Some((t, h))) if t >= b.tx => Some(h),
482 (Some(b), _) => Some(b.fn_hash),
483 (None, Some((_, h))) => Some(h),
484 (None, None) => None,
485 })
486}
487
488fn latest_bind<S: Store + ?Sized>(
490 store: &S,
491 sim: &Simulation,
492 name: &str,
493 class: bool,
494) -> Result<Option<String>, String> {
495 let stored: Option<String> = if class {
496 store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
497 } else {
498 store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
499 };
500 let binds = if class { &sim.classes } else { &sim.functions };
501 let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
502 Ok(match (stored, planned) {
503 (Some(a), Some(b)) => Some(a.max(b)),
504 (a, b) => a.or(b),
505 })
506}
507
508fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
510 if class {
511 Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
512 } else {
513 Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
514 || sim.classes.contains_key(name))
515 }
516}
517
518fn cell_latest<S: Store + ?Sized>(
520 reads: &mut Reads,
521 store: &S,
522 sim: &Simulation,
523 id: &str,
524) -> Result<Option<(String, String)>, String> {
525 let stored = reads.latest(store, id)?;
526 let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
527 Ok(match (stored, planned) {
528 (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
529 (a, b) => a.or(b),
530 })
531}
532
533fn cell_version_asof<S: Store + ?Sized>(
535 store: &S,
536 sim: &Simulation,
537 id: &str,
538 at: &str,
539) -> Result<Option<String>, String> {
540 let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
541 let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
542 let latest = match (stored, planned) {
543 (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
544 (a, b) => a.or(b),
545 };
546 Ok(latest.map(|x| x.1))
547}
548
549fn function_bytes_of<S: Store + ?Sized>(
551 reads: &mut Reads,
552 store: &S,
553 sim: &Simulation,
554 hash: &str,
555) -> Result<Option<Vec<u8>>, String> {
556 if let Some(b) = sim.descriptors.get(hash) {
557 return Ok(Some(b.clone()));
558 }
559 Ok(reads.descriptor(store, hash))
560}
561
562fn suspended<S: Store + ?Sized>(reads: &mut Reads, store: &S, host: &str, line: usize) -> Result<(), PublishError> {
564 match reads.namespace(store, host) {
565 Ok(Some(n)) if n.suspended_at.is_some() => {
566 Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
567 }
568 Ok(_) => Ok(()),
569 Err(e) => Err(PublishError::at(line, e)),
570 }
571}
572
573struct Bind {
575 line: usize,
576 name: String,
577}
578
579#[derive(Default)]
581struct Written {
582 namespaces: BTreeSet<String>,
583 starts: BTreeSet<String>,
584}
585
586impl Written {
587 fn bound(&mut self, name: &str, start: &str) {
588 self.namespaces.insert(namespace_of(name).to_string());
589 self.starts.insert(start.to_string());
590 }
591}
592
593struct Ctx<'a> {
595 host: &'a str,
596 owned: &'a [String],
597 key_from: &'a str,
599 now: &'a str,
601 base: &'a Path,
603}
604
605enum Op {
607 Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
609 Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
611 Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
613 Cell { tx: String, id: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
615 Catalog { entry: datumesh_store::CatalogEntry, dictionaries: Vec<(String, Vec<datumesh_store::KeyValue>)> },
617}
618
619fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
621 let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
622 let mut out = Vec::new();
623 for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
624 let n = i + 1;
625 let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
626 if !line.trim().is_empty() {
627 out.push((n, line));
628 }
629 }
630 Ok(out)
631}
632
633fn index_cell_values<S: Store + ?Sized>(store: &S, ops: &[Op]) -> Result<(), String> {
637 let mut seen = BTreeSet::new();
638 for op in ops {
639 if let Op::Cell { receipt, .. } = op {
640 if let Ok(r) = datumesh_core::Receipt::from_json(receipt) {
641 cell_string_values(&r, &mut seen);
642 }
643 }
644 }
645 store_cell_values(store, seen)
646}
647
648pub(crate) fn cell_string_values(r: &datumesh_core::Receipt, out: &mut BTreeSet<(String, String, String)>) {
650 let cell = r.cell();
651 for (key, arg) in &cell.args {
652 if let datumesh_core::Arg::Value(datumesh_core::Scalar::String(v)) = arg {
653 if !v.is_empty() && v.len() <= 1024 {
654 out.insert((cell.function_name(), key.clone(), v.clone()));
655 }
656 }
657 }
658}
659
660pub(crate) fn store_cell_values<S: Store + ?Sized>(
662 store: &S,
663 seen: BTreeSet<(String, String, String)>,
664) -> Result<(), String> {
665 for (owner, key, value) in seen {
666 if crate::find::has_value(store, &owner, &key, &value) {
667 continue;
668 }
669 let kv = datumesh_store::KeyValue { value, label: String::new(), aliases: Vec::new() };
670 store.upsert_key_values(&owner, &key, std::slice::from_ref(&kv)).map_err(|e| e.to_string())?;
671 }
672 Ok(())
673}
674
675fn is_cell_line(v: &Value) -> bool {
677 v.get("class").and_then(Value::as_str).is_none()
678 && v.get("descriptor").is_none()
679 && v.get("function").and_then(Value::as_str).is_none()
680 && v.get("formula").and_then(Value::as_str).is_none()
681 && v.get("cell").is_some()
682}
683
684struct Pending {
686 line: usize,
687 ns: String,
688 cell: Cell,
689 value: Scalar,
690 valid: Args,
691 tx: String,
692 asof: String,
693 issued: String,
694 has_tx: bool,
695 licence: String,
696 summary: String,
697 blob: Option<Vec<u8>>,
698 source: Option<Source>,
700}
701
702fn source_of_line(v: &Value) -> Result<Source, String> {
706 let Value::Object(m) = v else { return Err("source is an object".into()) };
707 for (k, x) in m {
708 if !matches!(k.as_str(), "name" | "url" | "retrieved" | "sha256" | "takeover") {
709 return Err(format!("source has no member {k}"));
710 }
711 if !matches!(x, Value::String(_)) {
712 return Err(format!("source.{k} is a string"));
713 }
714 }
715 let text = |k: &str| -> Result<&str, String> {
716 v.get(k).and_then(Value::as_str).ok_or_else(|| format!("source has no {k}"))
717 };
718 let mut out = Source::new(text("name")?, text("url")?, text("retrieved")?);
719 if let Some(h) = v.get("sha256").and_then(Value::as_str) {
720 out = out.with_sha256(h);
721 }
722 if let Some(t) = v.get("takeover").and_then(Value::as_str) {
723 out = out.with_takeover(t);
724 }
725 Ok(out)
726}
727
728impl Pending {
729 fn sign(self, key: &SigningKey, host: &str) -> Result<Signed, PublishError> {
731 let line = self.line;
732 let err = move |d: String| PublishError::at(line, d);
733 let b = ReceiptBuilder::new()
734 .cell(self.cell.clone())
735 .value(self.value)
736 .valid(self.valid)
737 .asof(self.asof.as_str())
738 .tx(self.tx.as_str())
739 .issued(self.issued.as_str())
740 .publisher(self.ns.as_str())
741 .signer_host(host)
742 .licence(self.licence.as_str())
743 .summary(self.summary.as_str());
744 let b = match self.source {
745 Some(s) => b.source(s),
746 None => b,
747 };
748 let r = b.sign(key).map_err(|e| err(e.to_string()))?;
749 let receipt = receipt_json(&r);
750 let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
751 let id = self.cell.id();
752 Ok(Signed { line, has_tx: self.has_tx, tx: self.tx, id, hash, receipt, blob: self.blob })
753 }
754}
755
756struct Signed {
758 line: usize,
759 has_tx: bool,
760 tx: String,
761 id: String,
762 hash: String,
763 receipt: Vec<u8>,
764 blob: Option<Vec<u8>>,
765}
766
767struct Planner<'a, S: Store + ?Sized> {
769 store: &'a S,
770 key: &'a SigningKey,
771 ctx: Ctx<'a>,
772 backfill: bool,
773 sim: Simulation,
774 reads: Reads,
775 ops: Vec<Op>,
776 binds: Vec<Bind>,
777 tests: Vec<(usize, TestSpec)>,
779 formulas: Vec<(usize, String, String)>,
781}
782
783impl<S: Store + ?Sized> Planner<'_, S> {
784 fn callers(&self) -> Callers<'_, S> {
785 Callers { store: self.store, sim: Some(&self.sim) }
786 }
787
788 fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
791 let mut cells: Vec<(usize, Value)> = Vec::new();
792 for (n, text) in lines {
793 let n = *n;
794 match json::parse(text.as_bytes()).map_err(|e| PublishError::at(n, e.to_string())) {
795 Ok(v) if is_cell_line(&v) => {
796 cells.push((n, v));
797 if cells.len() == CHUNK {
798 self.plan_cells(std::mem::take(&mut cells))?;
799 }
800 }
801 other => {
802 self.plan_cells(std::mem::take(&mut cells))?;
805 self.plan_line(n, &other?)?;
806 }
807 }
808 }
809 self.plan_cells(cells)
810 }
811
812 fn plan_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
814 let err = |d: String| PublishError::at(n, d);
815 if let Some(name) = v.get("class").and_then(Value::as_str) {
816 self.class_line(n, name, v)?;
817 } else if let Some(d) = v.get("descriptor") {
818 self.descriptor_line(n, d)?;
819 } else if let Some(name) = v.get("function").and_then(Value::as_str) {
820 let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
821 let start = bound_of(v, n, self.ctx.now)?;
822 let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
823 self.function(n, name, start, fd.hash(), fd.canonical())?;
824 } else if v.get("catalog").is_some() {
825 self.catalog_line(n, v)?;
826 } else if let Some(name) = v.get("formula").and_then(Value::as_str) {
827 let source =
828 v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
829 let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
830 let start = bound_of(v, n, self.ctx.now)?;
831 let def = datumesh_formula::definition::define_function(
832 name,
833 source,
834 sig,
835 &Default::default(),
836 &self.callers(),
837 &start,
838 )
839 .map_err(|e| err(format!("define {name}: {e}")))?;
840 let hash = def.fn_hash.clone();
841 self.function(n, name, start, def.fn_hash, def.bytes)?;
842 self.formulas.push((n, name.to_string(), hash));
843 } else if v.get("test").is_some() {
844 let spec = testrun::parse_test(v).map_err(err)?;
845 self.tests.push((n, spec));
846 } else {
847 return Err(err("neither a function nor a cell".into()));
848 }
849 Ok(())
850 }
851
852 fn catalog_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
856 let err = |d: String| PublishError::at(n, d);
857 let d = crate::catalog::parse(v, "catalog").map_err(err)?;
858 let host = host_of(&d.name);
859 if host != self.ctx.host && !self.ctx.owned.iter().any(|o| o == host) {
860 return Err(err(format!("{} is not under a host this mesh owns", d.name)));
861 }
862 let hash = match d.kind {
863 datumesh_store::CatalogKind::Class => current_class(self.store, &self.sim, &d.name).map_err(err)?,
864 _ => current_function(self.store, &self.sim, &d.name).map_err(err)?,
865 };
866 if hash.is_none() {
867 return Err(err(format!("{} is bound nowhere", d.name)));
868 }
869 let entry = d.entry(hash, datumesh_store::CatalogSource::Publisher, Some(self.key.key_id().to_string()));
870 self.ops.push(Op::Catalog { entry, dictionaries: d.dictionary });
871 Ok(())
872 }
873
874 fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
876 let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
877 if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
878 return Err(PublishError::at(
879 n,
880 "a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
881 ));
882 }
883 let (hash, bytes) = (fd.hash(), fd.canonical());
884 self.sim.descriptors.insert(hash.clone(), bytes.clone());
885 self.ops.push(Op::Descriptor { hash, bytes, counted: true });
886 Ok(())
887 }
888
889 fn function(
891 &mut self,
892 n: usize,
893 name: &str,
894 start: String,
895 hash: String,
896 bytes: Vec<u8>,
897 ) -> Result<(), PublishError> {
898 self.sim.descriptors.insert(hash.clone(), bytes.clone());
899 let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
900 let bind = current.as_deref() != Some(hash.as_str());
901 if bind {
902 self.check_bind(n, name, &start, false)?;
903 self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
904 }
905 self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
906 Ok(())
907 }
908
909 fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
910 let err = |d: String| PublishError::at(n, d);
911 let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
912 let start = bound_of(v, n, self.ctx.now)?;
913 let def = datumesh_formula::definition::define_class(
914 name,
915 &descriptor.to_jcs(),
916 definer(name, self.ctx.host, self.ctx.owned),
917 &self.callers(),
918 &start,
919 )
920 .map_err(|e| err(format!("define {name}: {e}")))?;
921 suspended(&mut self.reads, self.store, host_of(&def.descriptor.steward.namespace), n)?;
925 let hash = def.class_hash.clone();
926 let methods: Vec<(String, String)> =
927 def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
928 let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
929 if bind {
930 self.check_bind(n, name, &start, true)?;
931 for (mname, _) in &methods {
933 self.check_bind(n, mname, &start, false)?;
934 }
935 self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
936 for (mname, fh) in &methods {
937 self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
938 }
939 }
940 self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
941 self.ops.push(Op::Class {
942 name: name.to_string(),
943 start,
944 hash,
945 bytes: def.bytes,
946 bind,
947 methods: methods.into_iter().map(|(m, _)| m).collect(),
948 });
949 Ok(())
950 }
951
952 fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
954 if !self.backfill && start < self.ctx.now {
955 return Err(PublishError::typed(
956 line,
957 PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
958 ));
959 }
960 let err = |d: String| PublishError::at(line, d);
961 if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
962 let kind = if class { "function" } else { "class" };
963 return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
964 }
965 suspended(&mut self.reads, self.store, host_of(name), line)?;
966 if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
967 if start <= latest.as_str() {
968 return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
969 }
970 }
971 self.binds.push(Bind { line, name: name.into() });
972 Ok(())
973 }
974
975 fn plan_cells(&mut self, lines: Vec<(usize, Value)>) -> Result<(), PublishError> {
980 if lines.is_empty() {
981 return Ok(());
982 }
983 let mut pending = Vec::with_capacity(lines.len());
984 let mut failed = None;
985 for (n, v) in &lines {
986 match self.prepare_cell(*n, v) {
987 Ok(p) => pending.push(p),
988 Err(e) => {
989 failed = Some(e);
990 break;
991 }
992 }
993 }
994 let (key, host) = (self.key, self.ctx.host);
995 let signed = parallel_owned(pending, |p| p.sign(key, host));
996 let ids: Vec<&str> = signed.iter().flatten().map(|s| s.id.as_str()).collect();
997 self.reads.prefetch_latest(self.store, &ids);
998 let hashes: Vec<String> = signed
999 .iter()
1000 .flatten()
1001 .filter(|s| !s.has_tx)
1002 .filter_map(|s| self.reads.latest_of.get(&s.id).cloned().flatten())
1003 .map(|(_, h)| h)
1004 .collect();
1005 let hashes: Vec<&str> = hashes.iter().map(String::as_str).collect();
1006 self.reads.prefetch_receipts(self.store, &hashes);
1007 for s in signed {
1008 self.check_cell(s?)?;
1009 }
1010 match failed {
1011 Some(e) => Err(e),
1012 None => Ok(()),
1013 }
1014 }
1015
1016 fn prepare_cell(&mut self, n: usize, v: &Value) -> Result<Pending, PublishError> {
1019 let err = |d: String| PublishError::at(n, d);
1020 let cell_v = v.get("cell").ok_or_else(|| err("a cell without fn".into()))?;
1021 let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
1022 let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
1023 let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
1024 let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
1025 let (licence, summary) = (required("licence")?, required("summary")?);
1026 let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
1028 let has_tx = field("tx").is_some();
1029 let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
1030 if tx > asof {
1031 return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
1032 }
1033 let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
1034 if asof > issued {
1035 return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
1036 }
1037 let fn_hash = function_at(&mut self.reads, self.store, &self.sim, name, &tx)
1038 .map_err(err)?
1039 .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
1040 let d = function_bytes_of(&mut self.reads, self.store, &self.sim, &fn_hash)
1041 .map_err(err)?
1042 .ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
1043 let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
1044 let mut args = Args::new();
1045 for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
1046 args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
1047 }
1048 let valid: Args = args
1049 .iter()
1050 .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
1051 .map(|(k, a)| (k.clone(), a.clone()))
1052 .collect();
1053 let source = match v.get("source") {
1054 Some(sv) => Some(source_of_line(sv).map_err(err)?),
1055 None => None,
1056 };
1057 let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
1058 let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
1059 Some(file) => {
1060 let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
1061 let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
1062 let sha = sha256_hex(&bytes);
1063 let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
1064 .map_err(|e| err(e.to_string()))?;
1065 (scalar, Some(bytes))
1066 }
1067 None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
1068 };
1069 let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
1070 Ok(Pending {
1071 line: n,
1072 ns: ns.to_string(),
1073 cell,
1074 value,
1075 valid,
1076 tx,
1077 asof,
1078 issued,
1079 has_tx,
1080 licence,
1081 summary,
1082 blob,
1083 source,
1084 })
1085 }
1086
1087 fn check_cell(&mut self, s: Signed) -> Result<(), PublishError> {
1090 let line = s.line;
1091 let err = |d: String| PublishError::at(line, d);
1092 if !s.has_tx {
1097 if let Some((_, latest)) = cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1098 if let Some(old) = self.reads.receipt(self.store, &latest) {
1099 let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
1100 let new = json::parse(&s.receipt).map_err(|e| err(e.to_string()))?;
1101 if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
1102 return Ok(());
1103 }
1104 }
1105 }
1106 }
1107 let version = match cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
1108 Some((latest_tx, _)) if latest_tx.as_str() >= s.tx.as_str() => {
1109 if cell_version_asof(self.store, &self.sim, &s.id, &s.tx).map_err(err)?.as_deref()
1110 != Some(s.hash.as_str())
1111 {
1112 return Err(err(format!("{} has a version at or after {} that is not this receipt", s.id, s.tx)));
1113 }
1114 false
1115 }
1116 _ => {
1117 self.sim.versions.entry(s.id.clone()).or_default().push((s.tx.clone(), s.hash.clone()));
1118 true
1119 }
1120 };
1121 self.ops.push(Op::Cell { tx: s.tx, id: s.id, receipt: s.receipt, blob: s.blob, version });
1122 Ok(())
1123 }
1124}
1125
1126fn stored_value(signature: &Value) -> Value {
1128 Value::Object(vec![
1129 ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
1130 ("inputs".into(), Value::Array(vec![])),
1131 ("signature".into(), signature.clone()),
1132 ])
1133}
1134
1135fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
1138 for b in binds {
1139 let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
1140 if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
1141 return Err(refuse(format!("{} has a binding", b.name)));
1142 }
1143 let ns = namespace_of(&b.name);
1144 if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
1145 return Err(refuse(format!("{ns} has bindings")));
1146 }
1147 }
1148 Ok(())
1149}
1150
1151fn append_log(log: &Path, written: &Written) -> Result<(), String> {
1153 let line = Value::Object(vec![
1154 ("at".into(), s(&SystemClock.now())),
1155 ("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
1156 ("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
1157 ])
1158 .to_jcs();
1159 let mut opts = std::fs::OpenOptions::new();
1160 opts.create(true).append(true);
1161 #[cfg(unix)]
1162 {
1163 use std::os::unix::fs::OpenOptionsExt;
1164 opts.mode(0o600);
1165 }
1166 let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
1167 let mut bytes = line;
1168 bytes.push(b'\n');
1169 f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
1170}
1171
1172fn write<S: Store + ?Sized>(
1176 store: &S,
1177 ops: &[Op],
1178 cells: impl Fn(&S, &[Op], &mut Published) -> Result<(), String>,
1179) -> Result<(Published, Written), PublishError> {
1180 let mut out = Published::default();
1181 let mut written = Written::default();
1182 let err = |d: String| PublishError::at(0, d);
1183 let mut i = 0;
1184 while i < ops.len() {
1185 match &ops[i] {
1186 Op::Descriptor { hash, bytes, counted } => {
1187 let existed = store.descriptor(hash).is_some();
1188 store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1189 if *counted && !existed {
1190 out.descriptors += 1;
1191 }
1192 }
1193 Op::Function { name, start, hash, bytes, bind } => {
1194 store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1195 if *bind {
1196 store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1197 out.functions += 1;
1198 written.bound(name, start);
1199 }
1200 }
1201 Op::Class { name, start, hash, bytes, bind, methods } => {
1202 store.put_class(bytes).map_err(|e| err(e.to_string()))?;
1203 if *bind {
1204 store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1205 out.classes += 1;
1206 written.bound(name, start);
1207 for m in methods {
1208 written.bound(m, start);
1209 }
1210 }
1211 }
1212 Op::Catalog { entry, dictionaries } => {
1213 crate::catalog::store_publisher(store, entry, dictionaries).map_err(err)?;
1214 }
1215 Op::Cell { .. } => {
1216 let mut j = i;
1217 while j < ops.len() && j - i < CHUNK && matches!(ops[j], Op::Cell { .. }) {
1218 j += 1;
1219 }
1220 cells(store, &ops[i..j], &mut out).map_err(err)?;
1221 i = j;
1222 continue;
1223 }
1224 }
1225 i += 1;
1226 }
1227 Ok((out, written))
1228}
1229
1230fn write_cells_one_lane<S: Store + ?Sized>(store: &S, group: &[Op], out: &mut Published) -> Result<(), String> {
1234 let items = put_blobs_and_items(store, group, out)?;
1235 let failed = AtomicBool::new(false);
1236 let ks: Vec<usize> = (0..items.len()).collect();
1237 if let Some((_, detail)) = put_lane(store, &items, &ks, &failed) {
1238 return Err(detail);
1239 }
1240 count_cells(group, &items, out);
1241 Ok(())
1242}
1243
1244fn write_cells<S: Store + Sync + ?Sized>(
1250 store: &S,
1251 group: &[Op],
1252 lanes: usize,
1253 out: &mut Published,
1254) -> Result<(), String> {
1255 if lanes <= 1 {
1256 return write_cells_one_lane(store, group, out);
1257 }
1258 let items = put_blobs_and_items(store, group, out)?;
1259 let mut by_lane: Vec<Vec<usize>> = vec![Vec::new(); lanes];
1260 for (k, op) in group.iter().enumerate() {
1261 if let Op::Cell { id, .. } = op {
1262 by_lane[lane_of(id, lanes)].push(k);
1263 }
1264 }
1265 let failed = AtomicBool::new(false);
1266 let failures: Vec<(usize, String)> = std::thread::scope(|scope| {
1267 let handles: Vec<_> = by_lane
1268 .iter()
1269 .map(|ks| {
1270 let (items, failed) = (&items, &failed);
1271 scope.spawn(move || put_lane(store, items, ks, failed))
1272 })
1273 .collect();
1274 handles.into_iter().filter_map(|h| h.join().unwrap_or_else(|p| std::panic::resume_unwind(p))).collect()
1275 });
1276 if let Some((_, detail)) = failures.into_iter().min_by_key(|(k, _)| *k) {
1277 return Err(detail);
1278 }
1279 count_cells(group, &items, out);
1280 Ok(())
1281}
1282
1283fn put_blobs_and_items<'a, S: Store + ?Sized>(
1285 store: &S,
1286 group: &'a [Op],
1287 out: &mut Published,
1288) -> Result<Vec<CellItem<'a>>, String> {
1289 for op in group {
1290 if let Op::Cell { blob: Some(bytes), .. } = op {
1291 store.put_blob(bytes).map_err(|e| e.to_string())?;
1292 out.blobs += 1;
1293 out.blob_bytes += bytes.len() as u64;
1294 }
1295 }
1296 Ok(group
1297 .iter()
1298 .filter_map(|op| match op {
1299 Op::Cell { tx, id, receipt, version, .. } => {
1300 Some((receipt.as_slice(), if *version { Some((id.as_str(), tx.as_str())) } else { None }))
1301 }
1302 _ => None,
1303 })
1304 .collect())
1305}
1306
1307fn count_cells(group: &[Op], items: &[CellItem<'_>], out: &mut Published) {
1309 out.cells += group.len();
1310 out.receipts += group.len();
1311 out.versions += items.iter().filter(|(_, v)| v.is_some()).count();
1312}
1313
1314fn lane_of(id: &str, lanes: usize) -> usize {
1316 id.bytes().next().and_then(|b| (b as char).to_digit(16)).unwrap_or(0) as usize % lanes
1317}
1318
1319type CellItem<'a> = (&'a [u8], Option<(&'a str, &'a str)>);
1321
1322fn put_lane<S: Store + ?Sized>(
1326 store: &S,
1327 items: &[CellItem<'_>],
1328 ks: &[usize],
1329 failed: &AtomicBool,
1330) -> Option<(usize, String)> {
1331 for call in ks.chunks(CHUNK) {
1332 if failed.load(Ordering::SeqCst) {
1333 return None;
1334 }
1335 let writes: Vec<CellWrite<'_>> = call.iter().map(|&k| CellWrite::new(items[k].0, items[k].1)).collect();
1336 if let Err(e) = store.put_cells(&writes) {
1337 failed.store(true, Ordering::SeqCst);
1338 return Some((call[0], e.to_string()));
1339 }
1340 }
1341 None
1342}
1343
1344pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
1346 let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
1347 if let Ok(Value::Object(m)) = json::parse(r.payload()) {
1348 members.extend(m);
1349 }
1350 Value::Object(members).to_jcs()
1351}
1352
1353pub fn publish_stored<S: Store + ?Sized>(
1358 store: &S,
1359 key: &SigningKey,
1360 host: &str,
1361 path: &Path,
1362) -> Result<Published, PublishError> {
1363 publish_stored_in(store, key, host, &[host.to_string()], path)
1364}
1365
1366pub fn publish_stored_in<S: Store + ?Sized>(
1370 store: &S,
1371 key: &SigningKey,
1372 host: &str,
1373 owned: &[String],
1374 path: &Path,
1375) -> Result<Published, PublishError> {
1376 publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
1377}
1378
1379pub fn publish_stored_with<S: Store + ?Sized>(
1386 store: &S,
1387 key: &SigningKey,
1388 host: &str,
1389 owned: &[String],
1390 path: &Path,
1391 opts: &PublishOptions,
1392) -> Result<Published, PublishError> {
1393 publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1394 write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1395 })
1396}
1397
1398pub fn publish_stored_checked<S: Store + ?Sized>(
1405 store: &S,
1406 key: &SigningKey,
1407 host: &str,
1408 owned: &[String],
1409 path: &Path,
1410 opts: &PublishOptions,
1411) -> Result<Published, PublishError> {
1412 publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1413 write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1414 })
1415}
1416
1417pub fn publish_stored_lanes<S: Store + Sync + ?Sized>(
1422 store: &S,
1423 key: &SigningKey,
1424 host: &str,
1425 owned: &[String],
1426 path: &Path,
1427 opts: &PublishOptions,
1428) -> Result<Published, PublishError> {
1429 let lanes = publish_lanes()?;
1430 publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
1431 write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1432 })
1433}
1434
1435fn overlay_of(ops: &[Op]) -> Result<Overlay, PublishError> {
1437 let mut overlay = Overlay::default();
1438 for op in ops {
1439 match op {
1440 Op::Function { name, start, hash, bytes, bind } => {
1441 overlay.descriptors.insert(hash.clone(), bytes.clone());
1442 if *bind {
1443 overlay.functions.entry(name.clone()).or_default().push((start.clone(), hash.clone()));
1444 }
1445 }
1446 Op::Descriptor { hash, bytes, .. } => {
1447 overlay.descriptors.insert(hash.clone(), bytes.clone());
1448 }
1449 Op::Cell { receipt, .. } => {
1450 overlay.cell(receipt).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1451 }
1452 Op::Class { .. } | Op::Catalog { .. } => {}
1453 }
1454 }
1455 Ok(overlay)
1456}
1457
1458#[allow(clippy::too_many_arguments)]
1463pub(crate) fn check_tests<S: Store + ?Sized>(
1464 store: &S,
1465 overlay: &Overlay,
1466 tests: &[(usize, TestSpec)],
1467 formulas: &[(usize, String, String)],
1468 host: &str,
1469 now: &str,
1470 checked: bool,
1471) -> Result<Vec<RowWrite>, PublishError> {
1472 if formulas.is_empty() {
1473 return Ok(Vec::new());
1474 }
1475 let mut file_tests: Vec<(usize, TestSpec, String)> = Vec::new();
1476 for (n, spec) in tests {
1477 let at = spec.asof.clone().unwrap_or_else(|| now.to_string());
1478 match testrun::binding_of(store, overlay, &spec.function, &at) {
1479 Ok(Some((hash, _))) => file_tests.push((*n, spec.clone(), hash)),
1480 Ok(None) => {
1481 return Err(PublishError::at(
1482 *n,
1483 format!("a test of {}: no function of that name is bound", spec.function),
1484 ))
1485 }
1486 Err(e) => return Err(PublishError::at(*n, e)),
1487 }
1488 }
1489 let mut rows = Vec::new();
1490 for (line, name, hash) in formulas {
1491 let own: Vec<(usize, TestSpec)> =
1492 file_tests.iter().filter(|t| &t.2 == hash).map(|t| (t.0, t.1.clone())).collect();
1493 let held = testrun::held_tests(store, hash).map_err(|e| PublishError::at(*line, e))?;
1494 if !checked && own.is_empty() && held.is_empty() {
1495 continue;
1496 }
1497 let mut specs: Vec<(Option<usize>, TestSpec)> = own.into_iter().map(|(n, s)| (Some(n), s)).collect();
1498 for (_, spec) in held {
1499 if !specs.iter().any(|(_, s)| s.case == spec.case) {
1500 specs.push((None, spec));
1501 }
1502 }
1503 let ran: Vec<Ran> = specs.iter().map(|(_, s)| run_test(store, overlay, now, s)).collect();
1504 let mut run = testrun::Run::new(store, overlay, now);
1505 let built = run.built(name, hash).map_err(|e| PublishError::at(*line, format!("{name}: {}", e.0)))?;
1506 let Some(compiled) = built.compiled.as_ref() else { continue };
1507 if checked {
1508 let mut failing: Vec<(usize, String)> = Vec::new();
1509 for ((n, spec), r) in specs.iter().zip(&ran) {
1510 if let Some(n) = n {
1511 let (result, detail) = testrun::result_of(spec, r);
1512 if result != "pass" {
1513 failing.push((*n, format!("{name}: test {} {detail}", spec.case)));
1514 }
1515 }
1516 }
1517 failing.sort_by_key(|f| f.0);
1518 if let Some((n, detail)) = failing.into_iter().next() {
1519 return Err(PublishError::typed(n, PublishErrorKind::TestsFailing(detail)));
1520 }
1521 let refs: Vec<&Ran> = ran.iter().collect();
1522 let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1523 if !cov.uncovered.is_empty() || !cov.unexercised.is_empty() {
1524 let mut text = format!("{name} is missing tests:");
1525 if !cov.uncovered.is_empty() {
1526 let cases: Vec<&str> = cov.uncovered.iter().map(|c| c.0.as_str()).collect();
1527 text += &format!(" uncovered cases {};", cases.join(", "));
1528 }
1529 if !cov.unexercised.is_empty() {
1530 text += &format!(" unexercised branches {}", cov.unexercised.join(", "));
1531 }
1532 return Err(PublishError::typed(
1533 *line,
1534 PublishErrorKind::TestsMissing(text.trim_end_matches(';').to_string()),
1535 ));
1536 }
1537 }
1538 let refs: Vec<&Ran> = ran.iter().collect();
1539 let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
1540 for ((_, spec), r) in specs.iter().zip(&ran) {
1541 let (result, detail) = testrun::result_of(spec, r);
1542 rows.push(RowWrite {
1543 fn_hash: hash.clone(),
1544 case: spec.case.clone(),
1545 jcs: spec.jcs.clone(),
1546 signed_by: host.to_string(),
1547 result,
1548 detail,
1549 branches_total: cov.branches,
1550 branches_covered: cov.covered,
1551 });
1552 }
1553 }
1554 Ok(rows)
1555}
1556
1557pub fn publish_stored_checked_lanes<S: Store + Sync + ?Sized>(
1560 store: &S,
1561 key: &SigningKey,
1562 host: &str,
1563 owned: &[String],
1564 path: &Path,
1565 opts: &PublishOptions,
1566) -> Result<Published, PublishError> {
1567 let lanes = publish_lanes()?;
1568 publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
1569 write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1570 })
1571}
1572
1573#[allow(clippy::too_many_arguments)]
1575fn publish_planned<S: Store + ?Sized>(
1576 store: &S,
1577 key: &SigningKey,
1578 host: &str,
1579 owned: &[String],
1580 path: &Path,
1581 opts: &PublishOptions,
1582 checked: bool,
1583 write_ops: impl FnOnce(&S, &[Op]) -> Result<(Published, Written), PublishError>,
1584) -> Result<Published, PublishError> {
1585 let lines = read_lines(path)?;
1586 let key_from = store
1588 .namespace(host)
1589 .ok()
1590 .flatten()
1591 .and_then(|n| json::parse(&n.keydoc).ok())
1592 .and_then(|d| match d.get("keys") {
1593 Some(Value::Array(ks)) => ks
1594 .iter()
1595 .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
1596 .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
1597 _ => None,
1598 })
1599 .ok_or_else(|| {
1600 PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
1601 })?;
1602 let now = opts.clock.now();
1603 let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
1604 let mut planner = Planner {
1605 store,
1606 key,
1607 ctx,
1608 backfill: opts.backfill.is_some(),
1609 sim: Simulation::default(),
1610 reads: Reads::default(),
1611 ops: Vec::new(),
1612 binds: Vec::new(),
1613 tests: Vec::new(),
1614 formulas: Vec::new(),
1615 };
1616 planner.run(&lines)?;
1617 let Planner { ops, binds, tests, formulas, .. } = planner;
1618 if opts.backfill.is_some() {
1619 backfill_check(store, &binds)?;
1620 }
1621 let rows = check_tests(store, &overlay_of(&ops)?, &tests, &formulas, host, &now, checked)?;
1624 let (out, written) = write_ops(store, &ops)?;
1625 index_cell_values(store, &ops).map_err(|e| PublishError::at(0, format!("catalogue: {e}")))?;
1626 testrun::apply_rows(store, &rows, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1627 let rebound: Vec<String> = ops
1628 .iter()
1629 .filter_map(|op| match op {
1630 Op::Function { name, bind: true, .. } => Some(name.clone()),
1631 _ => None,
1632 })
1633 .collect();
1634 testrun::rerun_readers(store, &rebound, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
1635 if let Some(log) = &opts.backfill {
1636 if out.functions + out.classes + out.versions > 0 {
1637 append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
1638 }
1639 }
1640 Ok(out)
1641}