1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
59use std::io::{BufRead, Write};
60use std::path::{Path, PathBuf};
61use std::sync::atomic::{AtomicBool, Ordering};
62use std::sync::Arc;
63
64use datumesh_core::json::{self, Value};
65use datumesh_core::{
66 sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey, Type,
67};
68use datumesh_engine::{Clock, SystemClock};
69use datumesh_formula::definition::{Callee, Callees, ClassCallee};
70use datumesh_store::{Binding, CellWrite, Namespace, Store};
71
72use crate::config::ConfigError;
73
74const END: &str = "9999-12-31T23:59:59.999999Z";
76
77const CHUNK: usize = 1_000;
79
80const LANES_VAR: &str = "DATUMESH_PUBLISH_LANES";
82
83fn publish_lanes() -> Result<usize, PublishError> {
86 let bad = |detail: String| PublishError::at(0, ConfigError::new(LANES_VAR, detail).to_string());
87 match std::env::var(LANES_VAR) {
88 Err(std::env::VarError::NotPresent) => Ok(std::thread::available_parallelism().map_or(1, |n| n.get()).min(4)),
89 Ok(v) => match v.parse::<usize>() {
90 Ok(n) if n > 0 => Ok(n),
91 _ => Err(bad(format!("must be a positive integer, not {v:?}"))),
92 },
93 Err(e) => Err(bad(e.to_string())),
94 }
95}
96
97#[non_exhaustive]
99#[derive(Clone, Debug, Default, PartialEq, Eq)]
100pub struct Published {
101 pub functions: usize,
103 pub classes: usize,
105 pub cells: usize,
107 pub receipts: usize,
109 pub versions: usize,
111 pub blobs: usize,
113 pub blob_bytes: u64,
115 pub descriptors: u64,
117}
118
119#[derive(Clone)]
121pub struct PublishOptions {
122 pub clock: Arc<dyn Clock + Send + Sync>,
125 pub backfill: Option<PathBuf>,
129}
130
131impl PublishOptions {
132 pub fn system() -> PublishOptions {
134 PublishOptions { clock: Arc::new(SystemClock), backfill: None }
135 }
136}
137
138#[non_exhaustive]
140#[derive(Clone, Debug, PartialEq, Eq)]
141pub enum PublishErrorKind {
142 Line,
144 StartBeforeNow {
146 name: String,
148 start: String,
150 now: String,
152 },
153 Backfill(String),
155 OwnedVectorName(String),
157}
158
159impl std::fmt::Display for PublishErrorKind {
160 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
161 match self {
162 PublishErrorKind::Line => f.write_str("refused"),
163 PublishErrorKind::StartBeforeNow { name, start, now } => {
164 write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
165 }
166 PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
167 PublishErrorKind::OwnedVectorName(name) => {
168 write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
169 }
170 }
171 }
172}
173
174#[derive(Clone, Debug, PartialEq, Eq)]
176pub struct PublishError {
177 pub line: usize,
179 pub detail: String,
181 pub kind: PublishErrorKind,
183}
184
185impl PublishError {
186 fn at(line: usize, detail: impl Into<String>) -> PublishError {
187 PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
188 }
189
190 pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
191 PublishError { line, detail: kind.to_string(), kind }
192 }
193}
194
195impl std::fmt::Display for PublishError {
196 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
197 write!(f, "line {}: {}", self.line, self.detail)
198 }
199}
200
201impl std::error::Error for PublishError {}
202
203fn s(v: &str) -> Value {
204 Value::String(v.to_string())
205}
206
207fn namespace_of(name: &str) -> &str {
209 name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
210}
211
212fn host_of(name: &str) -> &str {
214 name.split('/').next().unwrap_or("")
215}
216
217fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
220 let host_of_name = name.split('/').next().unwrap_or("");
221 if owned.iter().any(|o| o == host_of_name) {
222 namespace_of(name)
223 } else {
224 host
225 }
226}
227
228fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
230 match v.get("bound") {
231 None => Ok(now.to_string()),
232 Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
233 }
234}
235
236#[derive(Default)]
239struct Simulation {
240 functions: BTreeMap<String, Vec<(String, String)>>,
242 classes: BTreeMap<String, Vec<(String, String)>>,
244 descriptors: BTreeMap<String, Vec<u8>>,
246 class_bytes: BTreeMap<String, Vec<u8>>,
248 versions: BTreeMap<String, Vec<(String, String)>>,
250}
251
252impl Simulation {
253 fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
255 binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
256 }
257}
258
259struct Callers<'a, S: Store + ?Sized> {
261 store: &'a S,
262 sim: Option<&'a Simulation>,
263}
264
265impl<S: Store + ?Sized> Callers<'_, S> {
266 fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
267 if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
268 return Ok(Some(b.clone()));
269 }
270 Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
271 }
272
273 fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
274 if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
275 return Ok(Some(b.clone()));
276 }
277 self.store.class(hash).map_err(|e| e.to_string())
278 }
279}
280
281impl<S: Store + ?Sized> Callees for Callers<'_, S> {
282 fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
283 let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
284 let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
285 let (fn_hash, start) = match (stored, simmed) {
286 (Some(b), Some((t, h))) if t >= b.tx => (h, t),
287 (Some(b), _) => (b.fn_hash, b.tx),
288 (None, Some((t, h))) => (h, t),
289 (None, None) => return Ok(None),
290 };
291 let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
292 let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
293 Ok(Some(Callee { fn_hash, start, descriptor }))
294 }
295
296 fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
297 let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
298 let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
299 let (class_hash, start) = match (stored, simmed) {
300 (Some(b), Some((t, h))) if t >= b.tx => (h, t),
301 (Some(b), _) => (b.class_hash, b.tx),
302 (None, Some((t, h))) => (h, t),
303 (None, None) => return Ok(None),
304 };
305 let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
306 let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
307 Ok(Some(ClassCallee { class_hash, start, descriptor }))
308 }
309
310 fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
311 let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
312 ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
313 }
314
315 fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
316 let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
317 json::parse(&bytes).map(Some).map_err(|e| e.to_string())
318 }
319}
320
321#[derive(Default)]
326struct Reads {
327 bindings: HashMap<(String, String), Result<Option<Binding>, String>>,
328 descriptors: HashMap<String, Option<Vec<u8>>>,
329 namespaces: HashMap<String, Result<Option<Namespace>, String>>,
330 latest_of: HashMap<String, Option<(String, String)>>,
332 receipt_of: HashMap<String, Option<Vec<u8>>>,
334}
335
336impl Reads {
337 fn binding<S: Store + ?Sized>(&mut self, store: &S, name: &str, at: &str) -> Result<Option<Binding>, String> {
338 let key = (name.to_string(), at.to_string());
339 if let Some(r) = self.bindings.get(&key) {
340 return r.clone();
341 }
342 let r = store.resolve_binding(name, at).map_err(|e| e.to_string());
343 self.bindings.insert(key, r.clone());
344 r
345 }
346
347 fn descriptor<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
348 if let Some(b) = self.descriptors.get(hash) {
349 return b.clone();
350 }
351 let b = store.descriptor(hash).map(|b| b.into_owned());
352 self.descriptors.insert(hash.to_string(), b.clone());
353 b
354 }
355
356 fn namespace<S: Store + ?Sized>(&mut self, store: &S, host: &str) -> Result<Option<Namespace>, String> {
357 if let Some(r) = self.namespaces.get(host) {
358 return r.clone();
359 }
360 let r = store.namespace(host).map_err(|e| e.to_string());
361 self.namespaces.insert(host.to_string(), r.clone());
362 r
363 }
364
365 fn latest<S: Store + ?Sized>(&mut self, store: &S, id: &str) -> Result<Option<(String, String)>, String> {
367 if let Some(l) = self.latest_of.get(id) {
368 return Ok(l.clone());
369 }
370 store.latest(id).map(|v| v.map(|v| (v.tx, v.receipt_hash))).map_err(|e| e.to_string())
371 }
372
373 fn receipt<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
375 if let Some(b) = self.receipt_of.get(hash) {
376 return b.clone();
377 }
378 store.receipt(hash).map(|b| b.into_owned())
379 }
380
381 fn prefetch_latest<S: Store + ?Sized>(&mut self, store: &S, ids: &[&str]) {
383 let mut seen = HashSet::new();
384 let missing: Vec<&str> =
385 ids.iter().copied().filter(|id| !self.latest_of.contains_key(*id) && seen.insert(*id)).collect();
386 if missing.is_empty() {
387 return;
388 }
389 if let Ok(found) = store.latest_many(&missing) {
390 for (id, v) in missing.iter().zip(found) {
391 self.latest_of.insert(id.to_string(), v.map(|v| (v.tx, v.receipt_hash)));
392 }
393 }
394 }
395
396 fn prefetch_receipts<S: Store + ?Sized>(&mut self, store: &S, hashes: &[&str]) {
398 let mut seen = HashSet::new();
399 let missing: Vec<&str> =
400 hashes.iter().copied().filter(|h| !self.receipt_of.contains_key(*h) && seen.insert(*h)).collect();
401 if missing.is_empty() {
402 return;
403 }
404 if let Ok(found) = store.receipts_many(&missing) {
405 for (h, b) in missing.iter().zip(found) {
406 self.receipt_of.insert(h.to_string(), b);
407 }
408 }
409 }
410}
411
412fn parallel_owned<T: Send, R: Send>(items: Vec<T>, f: impl Fn(T) -> R + Sync) -> Vec<R> {
414 let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(items.len());
415 if threads <= 1 {
416 return items.into_iter().map(f).collect();
417 }
418 let size = items.len().div_ceil(threads);
419 let mut parts: Vec<Vec<T>> = Vec::with_capacity(threads);
420 let mut rest = items.into_iter();
421 loop {
422 let part: Vec<T> = rest.by_ref().take(size).collect();
423 if part.is_empty() {
424 break;
425 }
426 parts.push(part);
427 }
428 std::thread::scope(|s| {
429 let handles: Vec<_> = parts
430 .into_iter()
431 .map(|part| {
432 let f = &f;
433 s.spawn(move || part.into_iter().map(f).collect::<Vec<R>>())
434 })
435 .collect();
436 handles.into_iter().flat_map(|h| h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect()
437 })
438}
439
440fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
442 if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
443 return Ok(Some(h));
444 }
445 store.resolve_name(name, END).map_err(|e| e.to_string())
446}
447
448fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
450 if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
451 return Ok(Some(h));
452 }
453 store.resolve_class(name, END).map_err(|e| e.to_string())
454}
455
456fn function_at<S: Store + ?Sized>(
458 reads: &mut Reads,
459 store: &S,
460 sim: &Simulation,
461 name: &str,
462 at: &str,
463) -> Result<Option<String>, String> {
464 let stored = reads.binding(store, name, at)?;
465 let simmed = Simulation::latest(&sim.functions, name, at);
466 Ok(match (stored, simmed) {
467 (Some(b), Some((t, h))) if t >= b.tx => Some(h),
468 (Some(b), _) => Some(b.fn_hash),
469 (None, Some((_, h))) => Some(h),
470 (None, None) => None,
471 })
472}
473
474fn latest_bind<S: Store + ?Sized>(
476 store: &S,
477 sim: &Simulation,
478 name: &str,
479 class: bool,
480) -> Result<Option<String>, String> {
481 let stored: Option<String> = if class {
482 store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
483 } else {
484 store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
485 };
486 let binds = if class { &sim.classes } else { &sim.functions };
487 let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
488 Ok(match (stored, planned) {
489 (Some(a), Some(b)) => Some(a.max(b)),
490 (a, b) => a.or(b),
491 })
492}
493
494fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
496 if class {
497 Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
498 } else {
499 Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
500 || sim.classes.contains_key(name))
501 }
502}
503
504fn cell_latest<S: Store + ?Sized>(
506 reads: &mut Reads,
507 store: &S,
508 sim: &Simulation,
509 id: &str,
510) -> Result<Option<(String, String)>, String> {
511 let stored = reads.latest(store, id)?;
512 let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
513 Ok(match (stored, planned) {
514 (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
515 (a, b) => a.or(b),
516 })
517}
518
519fn cell_version_asof<S: Store + ?Sized>(
521 store: &S,
522 sim: &Simulation,
523 id: &str,
524 at: &str,
525) -> Result<Option<String>, String> {
526 let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
527 let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
528 let latest = match (stored, planned) {
529 (Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
530 (a, b) => a.or(b),
531 };
532 Ok(latest.map(|x| x.1))
533}
534
535fn function_bytes_of<S: Store + ?Sized>(
537 reads: &mut Reads,
538 store: &S,
539 sim: &Simulation,
540 hash: &str,
541) -> Result<Option<Vec<u8>>, String> {
542 if let Some(b) = sim.descriptors.get(hash) {
543 return Ok(Some(b.clone()));
544 }
545 Ok(reads.descriptor(store, hash))
546}
547
548fn suspended<S: Store + ?Sized>(reads: &mut Reads, store: &S, host: &str, line: usize) -> Result<(), PublishError> {
550 match reads.namespace(store, host) {
551 Ok(Some(n)) if n.suspended_at.is_some() => {
552 Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
553 }
554 Ok(_) => Ok(()),
555 Err(e) => Err(PublishError::at(line, e)),
556 }
557}
558
559struct Bind {
561 line: usize,
562 name: String,
563}
564
565#[derive(Default)]
567struct Written {
568 namespaces: BTreeSet<String>,
569 starts: BTreeSet<String>,
570}
571
572impl Written {
573 fn bound(&mut self, name: &str, start: &str) {
574 self.namespaces.insert(namespace_of(name).to_string());
575 self.starts.insert(start.to_string());
576 }
577}
578
579struct Ctx<'a> {
581 host: &'a str,
582 owned: &'a [String],
583 key_from: &'a str,
585 now: &'a str,
587 base: &'a Path,
589}
590
591enum Op {
593 Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
595 Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
597 Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
599 Cell { tx: String, id: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
601}
602
603fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
605 let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
606 let mut out = Vec::new();
607 for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
608 let n = i + 1;
609 let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
610 if !line.trim().is_empty() {
611 out.push((n, line));
612 }
613 }
614 Ok(out)
615}
616
617fn is_cell_line(v: &Value) -> bool {
619 v.get("class").and_then(Value::as_str).is_none()
620 && v.get("descriptor").is_none()
621 && v.get("function").and_then(Value::as_str).is_none()
622 && v.get("formula").and_then(Value::as_str).is_none()
623 && v.get("cell").is_some()
624}
625
626struct Pending {
628 line: usize,
629 ns: String,
630 cell: Cell,
631 value: Scalar,
632 valid: Args,
633 tx: String,
634 asof: String,
635 issued: String,
636 has_tx: bool,
637 licence: String,
638 summary: String,
639 blob: Option<Vec<u8>>,
640}
641
642impl Pending {
643 fn sign(self, key: &SigningKey, host: &str) -> Result<Signed, PublishError> {
645 let line = self.line;
646 let err = move |d: String| PublishError::at(line, d);
647 let r = ReceiptBuilder::new()
648 .cell(self.cell.clone())
649 .value(self.value)
650 .valid(self.valid)
651 .asof(self.asof.as_str())
652 .tx(self.tx.as_str())
653 .issued(self.issued.as_str())
654 .publisher(self.ns.as_str())
655 .signer_host(host)
656 .licence(self.licence.as_str())
657 .summary(self.summary.as_str())
658 .sign(key)
659 .map_err(|e| err(e.to_string()))?;
660 let receipt = receipt_json(&r);
661 let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
662 let id = self.cell.id();
663 Ok(Signed { line, has_tx: self.has_tx, tx: self.tx, id, hash, receipt, blob: self.blob })
664 }
665}
666
667struct Signed {
669 line: usize,
670 has_tx: bool,
671 tx: String,
672 id: String,
673 hash: String,
674 receipt: Vec<u8>,
675 blob: Option<Vec<u8>>,
676}
677
678struct Planner<'a, S: Store + ?Sized> {
680 store: &'a S,
681 key: &'a SigningKey,
682 ctx: Ctx<'a>,
683 backfill: bool,
684 sim: Simulation,
685 reads: Reads,
686 ops: Vec<Op>,
687 binds: Vec<Bind>,
688}
689
690impl<S: Store + ?Sized> Planner<'_, S> {
691 fn callers(&self) -> Callers<'_, S> {
692 Callers { store: self.store, sim: Some(&self.sim) }
693 }
694
695 fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
698 let mut cells: Vec<(usize, Value)> = Vec::new();
699 for (n, text) in lines {
700 let n = *n;
701 match json::parse(text.as_bytes()).map_err(|e| PublishError::at(n, e.to_string())) {
702 Ok(v) if is_cell_line(&v) => {
703 cells.push((n, v));
704 if cells.len() == CHUNK {
705 self.plan_cells(std::mem::take(&mut cells))?;
706 }
707 }
708 other => {
709 self.plan_cells(std::mem::take(&mut cells))?;
712 self.plan_line(n, &other?)?;
713 }
714 }
715 }
716 self.plan_cells(cells)
717 }
718
719 fn plan_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
721 let err = |d: String| PublishError::at(n, d);
722 if let Some(name) = v.get("class").and_then(Value::as_str) {
723 self.class_line(n, name, v)?;
724 } else if let Some(d) = v.get("descriptor") {
725 self.descriptor_line(n, d)?;
726 } else if let Some(name) = v.get("function").and_then(Value::as_str) {
727 let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
728 let start = bound_of(v, n, self.ctx.now)?;
729 let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
730 self.function(n, name, start, fd.hash(), fd.canonical())?;
731 } else if let Some(name) = v.get("formula").and_then(Value::as_str) {
732 let source =
733 v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
734 let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
735 let start = bound_of(v, n, self.ctx.now)?;
736 let def = datumesh_formula::definition::define_function(
737 name,
738 source,
739 sig,
740 &Default::default(),
741 &self.callers(),
742 &start,
743 )
744 .map_err(|e| err(format!("define {name}: {e}")))?;
745 self.function(n, name, start, def.fn_hash, def.bytes)?;
746 } else {
747 return Err(err("neither a function nor a cell".into()));
748 }
749 Ok(())
750 }
751
752 fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
754 let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
755 if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
756 return Err(PublishError::at(
757 n,
758 "a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
759 ));
760 }
761 let (hash, bytes) = (fd.hash(), fd.canonical());
762 self.sim.descriptors.insert(hash.clone(), bytes.clone());
763 self.ops.push(Op::Descriptor { hash, bytes, counted: true });
764 Ok(())
765 }
766
767 fn function(
769 &mut self,
770 n: usize,
771 name: &str,
772 start: String,
773 hash: String,
774 bytes: Vec<u8>,
775 ) -> Result<(), PublishError> {
776 self.sim.descriptors.insert(hash.clone(), bytes.clone());
777 let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
778 let bind = current.as_deref() != Some(hash.as_str());
779 if bind {
780 self.check_bind(n, name, &start, false)?;
781 self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
782 }
783 self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
784 Ok(())
785 }
786
787 fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
788 let err = |d: String| PublishError::at(n, d);
789 let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
790 let start = bound_of(v, n, self.ctx.now)?;
791 let def = datumesh_formula::definition::define_class(
792 name,
793 &descriptor.to_jcs(),
794 definer(name, self.ctx.host, self.ctx.owned),
795 &self.callers(),
796 &start,
797 )
798 .map_err(|e| err(format!("define {name}: {e}")))?;
799 suspended(&mut self.reads, self.store, host_of(&def.descriptor.steward.namespace), n)?;
803 let hash = def.class_hash.clone();
804 let methods: Vec<(String, String)> =
805 def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
806 let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
807 if bind {
808 self.check_bind(n, name, &start, true)?;
809 for (mname, _) in &methods {
811 self.check_bind(n, mname, &start, false)?;
812 }
813 self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
814 for (mname, fh) in &methods {
815 self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
816 }
817 }
818 self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
819 self.ops.push(Op::Class {
820 name: name.to_string(),
821 start,
822 hash,
823 bytes: def.bytes,
824 bind,
825 methods: methods.into_iter().map(|(m, _)| m).collect(),
826 });
827 Ok(())
828 }
829
830 fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
832 if !self.backfill && start < self.ctx.now {
833 return Err(PublishError::typed(
834 line,
835 PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
836 ));
837 }
838 let err = |d: String| PublishError::at(line, d);
839 if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
840 let kind = if class { "function" } else { "class" };
841 return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
842 }
843 suspended(&mut self.reads, self.store, host_of(name), line)?;
844 if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
845 if start <= latest.as_str() {
846 return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
847 }
848 }
849 self.binds.push(Bind { line, name: name.into() });
850 Ok(())
851 }
852
853 fn plan_cells(&mut self, lines: Vec<(usize, Value)>) -> Result<(), PublishError> {
858 if lines.is_empty() {
859 return Ok(());
860 }
861 let mut pending = Vec::with_capacity(lines.len());
862 let mut failed = None;
863 for (n, v) in &lines {
864 match self.prepare_cell(*n, v) {
865 Ok(p) => pending.push(p),
866 Err(e) => {
867 failed = Some(e);
868 break;
869 }
870 }
871 }
872 let (key, host) = (self.key, self.ctx.host);
873 let signed = parallel_owned(pending, |p| p.sign(key, host));
874 let ids: Vec<&str> = signed.iter().flatten().map(|s| s.id.as_str()).collect();
875 self.reads.prefetch_latest(self.store, &ids);
876 let hashes: Vec<String> = signed
877 .iter()
878 .flatten()
879 .filter(|s| !s.has_tx)
880 .filter_map(|s| self.reads.latest_of.get(&s.id).cloned().flatten())
881 .map(|(_, h)| h)
882 .collect();
883 let hashes: Vec<&str> = hashes.iter().map(String::as_str).collect();
884 self.reads.prefetch_receipts(self.store, &hashes);
885 for s in signed {
886 self.check_cell(s?)?;
887 }
888 match failed {
889 Some(e) => Err(e),
890 None => Ok(()),
891 }
892 }
893
894 fn prepare_cell(&mut self, n: usize, v: &Value) -> Result<Pending, PublishError> {
897 let err = |d: String| PublishError::at(n, d);
898 let cell_v = v.get("cell").ok_or_else(|| err("a cell without fn".into()))?;
899 let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
900 let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
901 let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
902 let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
903 let (licence, summary) = (required("licence")?, required("summary")?);
904 let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
906 let has_tx = field("tx").is_some();
907 let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
908 if tx > asof {
909 return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
910 }
911 let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
912 if asof > issued {
913 return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
914 }
915 let fn_hash = function_at(&mut self.reads, self.store, &self.sim, name, &tx)
916 .map_err(err)?
917 .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
918 let d = function_bytes_of(&mut self.reads, self.store, &self.sim, &fn_hash)
919 .map_err(err)?
920 .ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
921 let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
922 let mut args = Args::new();
923 for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
924 args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
925 }
926 let valid: Args = args
927 .iter()
928 .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
929 .map(|(k, a)| (k.clone(), a.clone()))
930 .collect();
931 let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
932 let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
933 Some(file) => {
934 let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
935 let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
936 let sha = sha256_hex(&bytes);
937 let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
938 .map_err(|e| err(e.to_string()))?;
939 (scalar, Some(bytes))
940 }
941 None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
942 };
943 let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
944 Ok(Pending {
945 line: n,
946 ns: ns.to_string(),
947 cell,
948 value,
949 valid,
950 tx,
951 asof,
952 issued,
953 has_tx,
954 licence,
955 summary,
956 blob,
957 })
958 }
959
960 fn check_cell(&mut self, s: Signed) -> Result<(), PublishError> {
963 let line = s.line;
964 let err = |d: String| PublishError::at(line, d);
965 if !s.has_tx {
970 if let Some((_, latest)) = cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
971 if let Some(old) = self.reads.receipt(self.store, &latest) {
972 let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
973 let new = json::parse(&s.receipt).map_err(|e| err(e.to_string()))?;
974 if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
975 return Ok(());
976 }
977 }
978 }
979 }
980 let version = match cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
981 Some((latest_tx, _)) if latest_tx.as_str() >= s.tx.as_str() => {
982 if cell_version_asof(self.store, &self.sim, &s.id, &s.tx).map_err(err)?.as_deref()
983 != Some(s.hash.as_str())
984 {
985 return Err(err(format!("{} has a version at or after {} that is not this receipt", s.id, s.tx)));
986 }
987 false
988 }
989 _ => {
990 self.sim.versions.entry(s.id.clone()).or_default().push((s.tx.clone(), s.hash.clone()));
991 true
992 }
993 };
994 self.ops.push(Op::Cell { tx: s.tx, id: s.id, receipt: s.receipt, blob: s.blob, version });
995 Ok(())
996 }
997}
998
999fn stored_value(signature: &Value) -> Value {
1001 Value::Object(vec![
1002 ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
1003 ("inputs".into(), Value::Array(vec![])),
1004 ("signature".into(), signature.clone()),
1005 ])
1006}
1007
1008fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
1011 for b in binds {
1012 let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
1013 if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
1014 return Err(refuse(format!("{} has a binding", b.name)));
1015 }
1016 let ns = namespace_of(&b.name);
1017 if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
1018 return Err(refuse(format!("{ns} has bindings")));
1019 }
1020 }
1021 Ok(())
1022}
1023
1024fn append_log(log: &Path, written: &Written) -> Result<(), String> {
1026 let line = Value::Object(vec![
1027 ("at".into(), s(&SystemClock.now())),
1028 ("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
1029 ("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
1030 ])
1031 .to_jcs();
1032 let mut opts = std::fs::OpenOptions::new();
1033 opts.create(true).append(true);
1034 #[cfg(unix)]
1035 {
1036 use std::os::unix::fs::OpenOptionsExt;
1037 opts.mode(0o600);
1038 }
1039 let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
1040 let mut bytes = line;
1041 bytes.push(b'\n');
1042 f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
1043}
1044
1045fn write<S: Store + ?Sized>(
1049 store: &S,
1050 ops: &[Op],
1051 cells: impl Fn(&S, &[Op], &mut Published) -> Result<(), String>,
1052) -> Result<(Published, Written), PublishError> {
1053 let mut out = Published::default();
1054 let mut written = Written::default();
1055 let err = |d: String| PublishError::at(0, d);
1056 let mut i = 0;
1057 while i < ops.len() {
1058 match &ops[i] {
1059 Op::Descriptor { hash, bytes, counted } => {
1060 let existed = store.descriptor(hash).is_some();
1061 store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1062 if *counted && !existed {
1063 out.descriptors += 1;
1064 }
1065 }
1066 Op::Function { name, start, hash, bytes, bind } => {
1067 store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
1068 if *bind {
1069 store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1070 out.functions += 1;
1071 written.bound(name, start);
1072 }
1073 }
1074 Op::Class { name, start, hash, bytes, bind, methods } => {
1075 store.put_class(bytes).map_err(|e| err(e.to_string()))?;
1076 if *bind {
1077 store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
1078 out.classes += 1;
1079 written.bound(name, start);
1080 for m in methods {
1081 written.bound(m, start);
1082 }
1083 }
1084 }
1085 Op::Cell { .. } => {
1086 let mut j = i;
1087 while j < ops.len() && j - i < CHUNK && matches!(ops[j], Op::Cell { .. }) {
1088 j += 1;
1089 }
1090 cells(store, &ops[i..j], &mut out).map_err(err)?;
1091 i = j;
1092 continue;
1093 }
1094 }
1095 i += 1;
1096 }
1097 Ok((out, written))
1098}
1099
1100fn write_cells_one_lane<S: Store + ?Sized>(store: &S, group: &[Op], out: &mut Published) -> Result<(), String> {
1104 let items = put_blobs_and_items(store, group, out)?;
1105 let failed = AtomicBool::new(false);
1106 let ks: Vec<usize> = (0..items.len()).collect();
1107 if let Some((_, detail)) = put_lane(store, &items, &ks, &failed) {
1108 return Err(detail);
1109 }
1110 count_cells(group, &items, out);
1111 Ok(())
1112}
1113
1114fn write_cells<S: Store + Sync + ?Sized>(
1120 store: &S,
1121 group: &[Op],
1122 lanes: usize,
1123 out: &mut Published,
1124) -> Result<(), String> {
1125 if lanes <= 1 {
1126 return write_cells_one_lane(store, group, out);
1127 }
1128 let items = put_blobs_and_items(store, group, out)?;
1129 let mut by_lane: Vec<Vec<usize>> = vec![Vec::new(); lanes];
1130 for (k, op) in group.iter().enumerate() {
1131 if let Op::Cell { id, .. } = op {
1132 by_lane[lane_of(id, lanes)].push(k);
1133 }
1134 }
1135 let failed = AtomicBool::new(false);
1136 let failures: Vec<(usize, String)> = std::thread::scope(|scope| {
1137 let handles: Vec<_> = by_lane
1138 .iter()
1139 .map(|ks| {
1140 let (items, failed) = (&items, &failed);
1141 scope.spawn(move || put_lane(store, items, ks, failed))
1142 })
1143 .collect();
1144 handles.into_iter().filter_map(|h| h.join().unwrap_or_else(|p| std::panic::resume_unwind(p))).collect()
1145 });
1146 if let Some((_, detail)) = failures.into_iter().min_by_key(|(k, _)| *k) {
1147 return Err(detail);
1148 }
1149 count_cells(group, &items, out);
1150 Ok(())
1151}
1152
1153fn put_blobs_and_items<'a, S: Store + ?Sized>(
1155 store: &S,
1156 group: &'a [Op],
1157 out: &mut Published,
1158) -> Result<Vec<CellItem<'a>>, String> {
1159 for op in group {
1160 if let Op::Cell { blob: Some(bytes), .. } = op {
1161 store.put_blob(bytes).map_err(|e| e.to_string())?;
1162 out.blobs += 1;
1163 out.blob_bytes += bytes.len() as u64;
1164 }
1165 }
1166 Ok(group
1167 .iter()
1168 .filter_map(|op| match op {
1169 Op::Cell { tx, id, receipt, version, .. } => {
1170 Some((receipt.as_slice(), if *version { Some((id.as_str(), tx.as_str())) } else { None }))
1171 }
1172 _ => None,
1173 })
1174 .collect())
1175}
1176
1177fn count_cells(group: &[Op], items: &[CellItem<'_>], out: &mut Published) {
1179 out.cells += group.len();
1180 out.receipts += group.len();
1181 out.versions += items.iter().filter(|(_, v)| v.is_some()).count();
1182}
1183
1184fn lane_of(id: &str, lanes: usize) -> usize {
1186 id.bytes().next().and_then(|b| (b as char).to_digit(16)).unwrap_or(0) as usize % lanes
1187}
1188
1189type CellItem<'a> = (&'a [u8], Option<(&'a str, &'a str)>);
1191
1192fn put_lane<S: Store + ?Sized>(
1196 store: &S,
1197 items: &[CellItem<'_>],
1198 ks: &[usize],
1199 failed: &AtomicBool,
1200) -> Option<(usize, String)> {
1201 for call in ks.chunks(CHUNK) {
1202 if failed.load(Ordering::SeqCst) {
1203 return None;
1204 }
1205 let writes: Vec<CellWrite<'_>> = call.iter().map(|&k| CellWrite::new(items[k].0, items[k].1)).collect();
1206 if let Err(e) = store.put_cells(&writes) {
1207 failed.store(true, Ordering::SeqCst);
1208 return Some((call[0], e.to_string()));
1209 }
1210 }
1211 None
1212}
1213
1214pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
1216 let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
1217 if let Ok(Value::Object(m)) = json::parse(r.payload()) {
1218 members.extend(m);
1219 }
1220 Value::Object(members).to_jcs()
1221}
1222
1223pub fn publish_stored<S: Store + ?Sized>(
1228 store: &S,
1229 key: &SigningKey,
1230 host: &str,
1231 path: &Path,
1232) -> Result<Published, PublishError> {
1233 publish_stored_in(store, key, host, &[host.to_string()], path)
1234}
1235
1236pub fn publish_stored_in<S: Store + ?Sized>(
1240 store: &S,
1241 key: &SigningKey,
1242 host: &str,
1243 owned: &[String],
1244 path: &Path,
1245) -> Result<Published, PublishError> {
1246 publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
1247}
1248
1249pub fn publish_stored_with<S: Store + ?Sized>(
1256 store: &S,
1257 key: &SigningKey,
1258 host: &str,
1259 owned: &[String],
1260 path: &Path,
1261 opts: &PublishOptions,
1262) -> Result<Published, PublishError> {
1263 publish_planned(store, key, host, owned, path, opts, |s, ops| {
1264 write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
1265 })
1266}
1267
1268pub fn publish_stored_lanes<S: Store + Sync + ?Sized>(
1273 store: &S,
1274 key: &SigningKey,
1275 host: &str,
1276 owned: &[String],
1277 path: &Path,
1278 opts: &PublishOptions,
1279) -> Result<Published, PublishError> {
1280 let lanes = publish_lanes()?;
1281 publish_planned(store, key, host, owned, path, opts, |s, ops| {
1282 write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
1283 })
1284}
1285
1286fn publish_planned<S: Store + ?Sized>(
1288 store: &S,
1289 key: &SigningKey,
1290 host: &str,
1291 owned: &[String],
1292 path: &Path,
1293 opts: &PublishOptions,
1294 write_ops: impl FnOnce(&S, &[Op]) -> Result<(Published, Written), PublishError>,
1295) -> Result<Published, PublishError> {
1296 let lines = read_lines(path)?;
1297 let key_from = store
1299 .namespace(host)
1300 .ok()
1301 .flatten()
1302 .and_then(|n| json::parse(&n.keydoc).ok())
1303 .and_then(|d| match d.get("keys") {
1304 Some(Value::Array(ks)) => ks
1305 .iter()
1306 .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
1307 .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
1308 _ => None,
1309 })
1310 .ok_or_else(|| {
1311 PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
1312 })?;
1313 let now = opts.clock.now();
1314 let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
1315 let mut planner = Planner {
1316 store,
1317 key,
1318 ctx,
1319 backfill: opts.backfill.is_some(),
1320 sim: Simulation::default(),
1321 reads: Reads::default(),
1322 ops: Vec::new(),
1323 binds: Vec::new(),
1324 };
1325 planner.run(&lines)?;
1326 let Planner { ops, binds, .. } = planner;
1327 if opts.backfill.is_some() {
1328 backfill_check(store, &binds)?;
1329 }
1330 let (out, written) = write_ops(store, &ops)?;
1331 if let Some(log) = &opts.backfill {
1332 if out.functions + out.classes + out.versions > 0 {
1333 append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
1334 }
1335 }
1336 Ok(out)
1337}