1use std::collections::{BTreeMap, BTreeSet};
7
8use datumesh_core::json::{self, Value};
9use datumesh_core::{
10 keys_from_response, lexical::to_hex, verify, verify_ed25519, FunctionDescriptor, Keys, Receipt, SigningKey,
11};
12use datumesh_engine::Clock;
13use datumesh_store::Store;
14
15const END: &str = "9999-12-31T23:59:59.999999Z";
17
18const MEMBERS: [&str; 6] = ["bundle", "descriptors", "manifest", "names", "namespace", "receipts"];
20
21#[non_exhaustive]
23#[derive(Clone, Debug, PartialEq, Eq)]
24pub enum BundleError {
25 Format(String),
27 NotOwned(String),
29 Manifest(String),
32 Descriptor(String),
34 Receipt(String),
37 TxInFuture {
39 receipt: String,
41 tx: String,
43 now: String,
45 },
46 Bind(String),
48 Store(String),
50 Stopped(usize),
52 Tests(String),
55}
56
57impl std::fmt::Display for BundleError {
58 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
59 match self {
60 BundleError::Format(s) => write!(f, "bundle format: {s}"),
61 BundleError::NotOwned(s) => write!(f, "not owned: {s}"),
62 BundleError::Manifest(s) => write!(f, "manifest: {s}"),
63 BundleError::Descriptor(s) => write!(f, "descriptor: {s}"),
64 BundleError::Receipt(s) => write!(f, "receipt: {s}"),
65 BundleError::TxInFuture { receipt, tx, now } => {
66 write!(f, "receipt {receipt}: tx {tx} is later than the mesh clock {now}")
67 }
68 BundleError::Bind(s) => write!(f, "binding: {s}"),
69 BundleError::Store(s) => write!(f, "store: {s}"),
70 BundleError::Stopped(n) => write!(f, "stopped after write {n}"),
71 BundleError::Tests(s) => write!(f, "tests: {s}"),
72 }
73 }
74}
75
76impl std::error::Error for BundleError {}
77
78#[non_exhaustive]
80#[derive(Clone, Debug, PartialEq, Eq)]
81pub struct Imported {
82 pub namespace: String,
84 pub functions_bound: usize,
86 pub receipts: usize,
88 pub versions: usize,
90 pub bound_at: String,
92}
93
94impl Imported {
95 pub fn to_jcs(&self) -> Vec<u8> {
97 Value::Object(vec![
98 ("bound_at".into(), s(&self.bound_at)),
99 ("functions_bound".into(), number(self.functions_bound)),
100 ("namespace".into(), s(&self.namespace)),
101 ("receipts".into(), number(self.receipts)),
102 ("versions".into(), number(self.versions)),
103 ])
104 .to_jcs()
105 }
106}
107
108fn s(v: &str) -> Value {
109 Value::String(v.to_string())
110}
111
112fn number(n: usize) -> Value {
113 json::parse(n.to_string().as_bytes()).unwrap_or(Value::Null)
114}
115
116fn is_hex(text: &str, len: usize) -> bool {
117 text.len() == len && text.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
118}
119
120fn unhex(text: &str) -> Vec<u8> {
121 (0..text.len()).step_by(2).filter_map(|i| u8::from_str_radix(text.get(i..i + 2)?, 16).ok()).collect()
122}
123
124fn host_of(namespace: &str) -> &str {
126 namespace.split('/').next().unwrap_or("")
127}
128
129fn name_value(name: &str, fn_hash: &str) -> Value {
131 Value::Object(vec![("fn_hash".into(), s(fn_hash)), ("name".into(), s(name))])
132}
133
134pub fn manifest_bytes(namespace: &str, names: &[(String, String)], receipts: &[String]) -> Vec<u8> {
137 manifest_bytes_with(namespace, names, receipts, None, None)
138}
139
140fn manifest_bytes_with(
143 namespace: &str,
144 names: &[(String, String)],
145 receipts: &[String],
146 catalog: Option<&Value>,
147 tests: Option<&Value>,
148) -> Vec<u8> {
149 let mut members = vec![
150 ("names".into(), Value::Array(names.iter().map(|(n, h)| name_value(n, h)).collect())),
151 ("namespace".into(), s(namespace)),
152 ("receipts".into(), Value::Array(receipts.iter().map(|h| s(h)).collect())),
153 ];
154 if let Some(c) = catalog {
155 members.push(("catalog".into(), c.clone()));
156 }
157 if let Some(t) = tests {
158 members.push(("tests".into(), t.clone()));
159 }
160 Value::Object(members).to_jcs()
161}
162
163fn sign(seed: &[u8; 32], message: &[u8]) -> [u8; 64] {
167 use ed25519_dalek::Signer;
168 ed25519_dalek::SigningKey::from_bytes(seed).sign(message).to_bytes()
169}
170
171pub fn build(
176 namespace: &str,
177 descriptors: Vec<Vec<u8>>,
178 names: Vec<(String, String)>,
179 receipts: Vec<Vec<u8>>,
180 seed: &[u8; 32],
181) -> Result<Vec<u8>, BundleError> {
182 build_with_catalog(namespace, descriptors, names, receipts, Vec::new(), seed)
183}
184
185pub fn build_with_tests(
190 namespace: &str,
191 descriptors: Vec<Vec<u8>>,
192 names: Vec<(String, String)>,
193 receipts: Vec<Vec<u8>>,
194 catalog: Vec<Vec<u8>>,
195 tests: Vec<Vec<u8>>,
196 seed: &[u8; 32],
197) -> Result<Vec<u8>, BundleError> {
198 let mut entries: Vec<(String, String, Value)> = Vec::new();
199 for t in tests {
200 let v = json::parse(&t).map_err(|e| BundleError::Format(format!("test: {e}")))?;
201 let spec = crate::testrun::parse_test(&v).map_err(|e| BundleError::Format(format!("test: {e}")))?;
202 entries.push((spec.function.clone(), spec.case.clone(), v));
203 }
204 entries.sort_by(|a, b| (a.0.as_str(), a.1.as_str()).cmp(&(b.0.as_str(), b.1.as_str())));
205 let tests_value = (!entries.is_empty()).then(|| Value::Array(entries.into_iter().map(|(_, _, v)| v).collect()));
206 build_inner(namespace, descriptors, names, receipts, catalog, tests_value, seed)
207}
208
209pub fn build_with_catalog(
215 namespace: &str,
216 descriptors: Vec<Vec<u8>>,
217 names: Vec<(String, String)>,
218 receipts: Vec<Vec<u8>>,
219 catalog: Vec<Vec<u8>>,
220 seed: &[u8; 32],
221) -> Result<Vec<u8>, BundleError> {
222 build_inner(namespace, descriptors, names, receipts, catalog, None, seed)
223}
224
225fn build_inner(
226 namespace: &str,
227 descriptors: Vec<Vec<u8>>,
228 names: Vec<(String, String)>,
229 receipts: Vec<Vec<u8>>,
230 catalog: Vec<Vec<u8>>,
231 tests: Option<Value>,
232 seed: &[u8; 32],
233) -> Result<Vec<u8>, BundleError> {
234 let mut catalog_entries: Vec<(String, Value)> = Vec::new();
235 for c in catalog {
236 let v = json::parse(&c).map_err(|e| BundleError::Format(format!("catalog: {e}")))?;
237 let d = crate::catalog::parse(&v, "name").map_err(|e| BundleError::Format(format!("catalog: {e}")))?;
238 catalog_entries.push((d.name.clone(), v));
239 }
240 catalog_entries.sort_by(|a, b| a.0.cmp(&b.0));
241 let catalog_value =
242 (!catalog_entries.is_empty()).then(|| Value::Array(catalog_entries.into_iter().map(|(_, v)| v).collect()));
243 let mut names = names;
244 names.sort();
245 let mut signed: Vec<(String, Value)> = Vec::new();
246 for r in receipts {
247 let hash = Receipt::from_json(&r).map_err(|e| BundleError::Format(format!("receipt: {e}")))?.hash().to_string();
248 let v = json::parse(&r).map_err(|e| BundleError::Format(format!("receipt: {e}")))?;
249 signed.push((hash, v));
250 }
251 signed.sort_by(|a, b| a.0.cmp(&b.0));
252 let hashes: Vec<String> = signed.iter().map(|(h, _)| h.clone()).collect();
253 let sig = sign(seed, &manifest_bytes_with(namespace, &names, &hashes, catalog_value.as_ref(), tests.as_ref()));
254 let key = SigningKey::from_seed(seed);
255 let descriptors = descriptors
256 .iter()
257 .map(|d| json::parse(d).map_err(|e| BundleError::Format(format!("descriptor: {e}"))))
258 .collect::<Result<Vec<Value>, _>>()?;
259 let mut members = vec![("bundle".into(), number(1))];
260 if let Some(c) = catalog_value {
261 members.push(("catalog".into(), c));
262 }
263 if let Some(t) = tests {
264 members.push(("tests".into(), t));
265 }
266 members.extend([
267 ("descriptors".into(), Value::Array(descriptors)),
268 (
269 "manifest".into(),
270 Value::Object(vec![("key_id".into(), s(key.key_id())), ("signature".into(), s(&to_hex(&sig)))]),
271 ),
272 ("names".into(), Value::Array(names.iter().map(|(n, h)| name_value(n, h)).collect())),
273 ("namespace".into(), s(namespace)),
274 ("receipts".into(), Value::Array(signed.into_iter().map(|(_, v)| v).collect())),
275 ]);
276 Ok(Value::Object(members).to_jcs())
277}
278
279fn member_names(v: &Value) -> Option<Vec<&str>> {
281 let mut keys: Vec<&str> = v.as_object()?.iter().map(|(k, _)| k.as_str()).collect();
282 keys.sort_unstable();
283 Some(keys)
284}
285
286fn text<'a>(v: &'a Value, key: &str) -> Result<&'a str, BundleError> {
288 v.get(key).and_then(Value::as_str).ok_or_else(|| BundleError::Format(format!("{key} is not a string")))
289}
290
291pub fn import_signed<S: Store + ?Sized>(
298 store: &S,
299 host: &str,
300 owned: &[String],
301 clock: &dyn Clock,
302 bytes: &[u8],
303) -> Result<Imported, BundleError> {
304 import_signed_with(store, host, owned, clock, bytes, None)
305}
306
307#[doc(hidden)]
311pub fn import_signed_with<S: Store + ?Sized>(
312 store: &S,
313 host: &str,
314 owned: &[String],
315 clock: &dyn Clock,
316 bytes: &[u8],
317 stop_after: Option<usize>,
318) -> Result<Imported, BundleError> {
319 let now = clock.now();
320 let doc = json::parse(bytes).map_err(|e| BundleError::Format(format!("not JSON: {e}")))?;
321
322 let mut keys = member_names(&doc).ok_or_else(|| BundleError::Format("not an object".into()))?;
324 keys.sort_unstable();
325 let has_catalog = keys.contains(&"catalog");
326 let has_tests = keys.contains(&"tests");
327 keys.retain(|k| *k != "catalog" && *k != "tests");
328 let mut want = MEMBERS;
329 want.sort_unstable();
330 if keys != want {
331 return Err(BundleError::Format("the members are not exactly those of a bundle".into()));
332 }
333 if doc.get("bundle").and_then(Value::as_number_text) != Some("1") {
334 return Err(BundleError::Format("bundle is not 1".into()));
335 }
336 let namespace = text(&doc, "namespace")?.to_string();
337 if namespace.split('/').count() < 2 {
338 return Err(BundleError::Format("namespace has no host and name".into()));
339 }
340 let manifest = doc.get("manifest").ok_or_else(|| BundleError::Format("no manifest".into()))?;
341 if member_names(manifest) != Some(vec!["key_id", "signature"]) {
342 return Err(BundleError::Format("the manifest's members are not key_id and signature".into()));
343 }
344 let key_id = text(manifest, "key_id")?.to_string();
345 let signature = text(manifest, "signature")?.to_string();
346 if !is_hex(&key_id, 16) || !is_hex(&signature, 128) {
347 return Err(BundleError::Format("the manifest's key_id or signature is not hex".into()));
348 }
349 let mut names: Vec<(String, String)> = Vec::new();
350 let Some(Value::Array(name_vs)) = doc.get("names") else {
351 return Err(BundleError::Format("names is not an array".into()));
352 };
353 for n in name_vs {
354 if member_names(n) != Some(vec!["fn_hash", "name"]) {
355 return Err(BundleError::Format("a name's members are not fn_hash and name".into()));
356 }
357 let fh = text(n, "fn_hash")?;
358 if !is_hex(fh, 64) {
359 return Err(BundleError::Format(format!("fn_hash {fh} is not 64 hex digits")));
360 }
361 names.push((text(n, "name")?.to_string(), fh.to_string()));
362 }
363 if names.windows(2).any(|w| w[0].0 >= w[1].0) {
364 return Err(BundleError::Format("names are not in ascending order".into()));
365 }
366 let Some(Value::Array(desc_vs)) = doc.get("descriptors") else {
367 return Err(BundleError::Format("descriptors is not an array".into()));
368 };
369 let Some(Value::Array(receipt_vs)) = doc.get("receipts") else {
370 return Err(BundleError::Format("receipts is not an array".into()));
371 };
372 let mut receipts: Vec<(Vec<u8>, Receipt)> = Vec::new();
374 for r in receipt_vs {
375 let rb = r.to_jcs();
376 let parsed = Receipt::from_json(&rb).map_err(|e| BundleError::Receipt(format!("unreadable: {e}")))?;
377 receipts.push((rb, parsed));
378 }
379 if receipts.windows(2).any(|w| w[0].1.hash() >= w[1].1.hash()) {
380 return Err(BundleError::Format("receipts are not in ascending order of their hashes".into()));
381 }
382 let hashes: Vec<String> = receipts.iter().map(|(_, r)| r.hash().to_string()).collect();
383 let mut catalog_entries: Vec<(crate::catalog::Described, Option<String>)> = Vec::new();
385 let catalog_doc = if has_catalog { doc.get("catalog").cloned() } else { None };
386 if let Some(Value::Array(items)) = &catalog_doc {
387 for item in items {
388 let d = crate::catalog::parse(item, "name").map_err(|e| BundleError::Format(format!("catalog: {e}")))?;
389 let hash =
390 match d.kind {
391 datumesh_store::CatalogKind::Class => None,
392 _ => {
393 Some(names.iter().find(|(n, _)| *n == d.name).map(|(_, h)| h.clone()).ok_or_else(|| {
394 BundleError::Format(format!("catalog: {} is not among the names", d.name))
395 })?)
396 }
397 };
398 catalog_entries.push((d, hash));
399 }
400 } else if has_catalog {
401 return Err(BundleError::Format("catalog is not an array".into()));
402 }
403
404 let ns_host = host_of(&namespace);
406 if ns_host != host && !owned.iter().any(|o| o == ns_host) {
407 return Err(BundleError::NotOwned(format!("{ns_host} is neither this mesh's host nor owned")));
408 }
409 if let Some((n, _)) = names.iter().find(|(n, _)| !n.starts_with(&format!("{namespace}/"))) {
410 return Err(BundleError::NotOwned(format!("{n} is not under {namespace}/")));
411 }
412 if let Some((d, _)) = catalog_entries.iter().find(|(d, _)| !d.name.starts_with(&format!("{namespace}/"))) {
413 return Err(BundleError::NotOwned(format!("catalog: {} is not under {namespace}/", d.name)));
414 }
415
416 let keydoc = store
418 .namespace(ns_host)
419 .map_err(|e| BundleError::Store(e.to_string()))?
420 .ok_or_else(|| BundleError::Manifest(format!("the store holds no key document of {ns_host}")))?
421 .keydoc;
422 let kd = keys_from_response(ns_host, 200, Some(&keydoc)).map_err(|e| BundleError::Manifest(e.to_string()))?;
423 let key = kd
424 .key(&key_id)
425 .ok_or_else(|| BundleError::Manifest(format!("key {key_id} is not in the key document of {ns_host}")))?;
426 if !key.usable_at(&now) {
427 return Err(BundleError::Manifest(format!("key {key_id} does not hold at {now}")));
428 }
429 let mut keys_set = Keys::new();
430 keys_set.insert(kd.clone());
431 let sig: [u8; 64] = unhex(&signature).try_into().map_err(|_| BundleError::Format("signature length".into()))?;
432 let tests_doc = if has_tests { doc.get("tests").cloned() } else { None };
433 if tests_doc.as_ref().is_some_and(|t| !matches!(t, Value::Array(_))) {
434 return Err(BundleError::Format("tests is not an array".into()));
435 }
436 if !verify_ed25519(
437 &key.public,
438 &manifest_bytes_with(&namespace, &names, &hashes, catalog_doc.as_ref(), tests_doc.as_ref()),
439 &sig,
440 ) {
441 return Err(BundleError::Manifest("the signature does not verify".into()));
442 }
443
444 let mut descriptors: BTreeMap<String, Vec<u8>> = BTreeMap::new();
446 for d in desc_vs {
447 let db = d.to_jcs();
448 let fd = FunctionDescriptor::from_json(&db).map_err(|e| BundleError::Descriptor(e.to_string()))?;
449 if fd.canonical() != db {
450 return Err(BundleError::Descriptor("a descriptor is not canonical".into()));
451 }
452 descriptors.insert(fd.hash(), db);
453 }
454 for (name, fh) in &names {
455 if !descriptors.contains_key(fh) && store.descriptor(fh).is_none() {
456 return Err(BundleError::Descriptor(format!(
457 "{name}: descriptor {fh} is neither in the bundle nor stored"
458 )));
459 }
460 }
461
462 for (rb, r) in &receipts {
464 if !r.inputs().is_empty() {
465 return Err(BundleError::Receipt(format!("{}: a computed receipt is not imported", r.hash())));
466 }
467 let fname = r.cell().function_name();
468 let Some((_, fh)) = names.iter().find(|(n, _)| *n == fname) else {
469 return Err(BundleError::Receipt(format!("{}: {fname} is not among the names", r.hash())));
470 };
471 if r.cell().fn_hash != *fh {
472 return Err(BundleError::Receipt(format!("{}: the function hash differs from {fname}'s", r.hash())));
473 }
474 let out = verify(rb, &keys_set, None, None);
475 if !out.ok {
476 let why = out.error().map(|e| e.to_string()).unwrap_or_else(|| "does not verify".into());
477 return Err(BundleError::Receipt(format!("{}: {why}", r.hash())));
478 }
479 }
480
481 for (_, r) in &receipts {
483 if r.tx() > now.as_str() {
484 return Err(BundleError::TxInFuture {
485 receipt: r.hash().to_string(),
486 tx: r.tx().to_string(),
487 now: now.clone(),
488 });
489 }
490 }
491
492 let mut overlay = crate::testrun::Overlay::default();
494 for (name, fh) in &names {
495 overlay.functions.entry(name.clone()).or_default().push((now.clone(), fh.clone()));
496 }
497 for (fh, db) in &descriptors {
498 overlay.descriptors.insert(fh.clone(), db.clone());
499 }
500 for (rb, _) in &receipts {
501 overlay.cell(rb).map_err(BundleError::Receipt)?;
502 }
503 let mut formulas = Vec::new();
504 for (name, fh) in &names {
505 let bytes = match descriptors.get(fh) {
506 Some(b) => Some(b.clone()),
507 None => store.descriptor(fh).map(|b| b.into_owned()),
508 };
509 let Some(bytes) = bytes else { continue };
510 let fd = FunctionDescriptor::from_json(&bytes).map_err(|e| BundleError::Descriptor(e.to_string()))?;
511 if fd.lang != "stored" {
512 formulas.push((0usize, name.clone(), fh.clone()));
513 }
514 }
515 let mut test_specs = Vec::new();
516 for (i, t) in tests_doc
517 .iter()
518 .flat_map(|v| match v {
519 Value::Array(a) => a.iter().collect::<Vec<_>>(),
520 _ => Vec::new(),
521 })
522 .enumerate()
523 {
524 let spec = crate::testrun::parse_test(t).map_err(|e| BundleError::Format(format!("test: {e}")))?;
525 test_specs.push((i + 1, spec));
526 }
527 let test_rows = crate::publish::check_tests(store, &overlay, &test_specs, &formulas, host, &now, true)
528 .map_err(|e| BundleError::Tests(e.detail))?;
529 let mut binds: Vec<&(String, String)> = Vec::new();
531 for pair in &names {
532 let (name, fh) = pair;
533 match store.resolve_binding(name, END).map_err(|e| BundleError::Store(e.to_string()))? {
534 Some(b) if b.fn_hash == *fh => {}
535 Some(b) if b.tx.as_str() >= now.as_str() => {
536 return Err(BundleError::Bind(format!("{name} is bound from {} (at or after the clock)", b.tx)));
537 }
538 _ => binds.push(pair),
539 }
540 }
541 let mut new_versions: Vec<(String, String, String)> = Vec::new(); for (_, r) in &receipts {
543 let id = r.cell().id();
544 let tx = r.tx();
545 match store.version_asof(&id, tx).map_err(|e| BundleError::Store(e.to_string()))? {
546 Some(v) if v.tx == tx => {
547 if v.receipt_hash != r.hash() {
548 return Err(BundleError::Store(format!("{id} has another version at {tx}")));
549 }
550 continue;
551 }
552 _ => {}
553 }
554 if let Some(l) = store.latest(&id).map_err(|e| BundleError::Store(e.to_string()))? {
555 if l.tx.as_str() >= tx {
556 return Err(BundleError::Store(format!("{id} has a version at or after {tx}")));
557 }
558 }
559 new_versions.push((id, tx.to_string(), r.hash().to_string()));
560 }
561
562 let mut writes = 0usize;
565 let mut step = || -> Result<(), BundleError> {
566 writes += 1;
567 if stop_after == Some(writes) {
568 return Err(BundleError::Stopped(writes));
569 }
570 Ok(())
571 };
572 for db in descriptors.values() {
573 store.put_descriptor(db).map_err(|e| BundleError::Store(e.to_string()))?;
574 step()?;
575 }
576 for (rb, _) in &receipts {
577 store.put_receipt(rb).map_err(|e| BundleError::Store(e.to_string()))?;
578 step()?;
579 }
580 for (id, tx, h) in &new_versions {
581 store.put_version(id, tx, h).map_err(|e| BundleError::Store(e.to_string()))?;
582 step()?;
583 }
584 for (name, fh) in &binds {
585 store.bind_name(name, fh, &now).map_err(|e| BundleError::Bind(format!("{name}: {e}")))?;
586 step()?;
587 }
588 crate::testrun::apply_rows(store, &test_rows, &now).map_err(BundleError::Store)?;
589 let rebound: Vec<String> = binds.iter().map(|(n, _)| (*n).clone()).collect();
590 crate::testrun::rerun_readers(store, &rebound, &now).map_err(BundleError::Store)?;
591 for (d, hash) in &catalog_entries {
593 let entry = d.entry(hash.clone(), datumesh_store::CatalogSource::Publisher, Some(key_id.clone()));
594 crate::catalog::store_publisher(store, &entry, &d.dictionary).map_err(BundleError::Store)?;
595 }
596 let mut cell_values = BTreeSet::new();
597 for (_, r) in &receipts {
598 crate::publish::cell_string_values(r, &mut cell_values);
599 }
600 crate::publish::store_cell_values(store, cell_values).map_err(BundleError::Store)?;
601 Ok(Imported {
602 namespace,
603 functions_bound: binds.len(),
604 receipts: receipts.len(),
605 versions: new_versions.len(),
606 bound_at: now,
607 })
608}
609
610#[cfg(test)]
611mod tests {
612 use super::*;
613
614 #[test]
616 fn rfc8032_test_1() {
617 let seed: [u8; 32] =
618 unhex("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60").try_into().unwrap();
619 let sig = sign(&seed, b"");
620 assert_eq!(
621 to_hex(&sig),
622 "e5564300c360ac729086e2cc806e828a84877f1eb8e5d974d873e065224901555fb8821590a33bacc61e39701cf9b46bd25bf5f0595bbe24655141438e7a100b"
623 );
624 }
625
626 #[test]
628 fn a_signature_verifies_under_the_seed_key() {
629 let seed = [7u8; 32];
630 let message = b"the manifest bytes";
631 let sig = sign(&seed, message);
632 let public = SigningKey::from_seed(&seed).public_key();
633 assert!(verify_ed25519(&public, message, &sig));
634 assert!(!verify_ed25519(&public, b"other bytes", &sig));
635 }
636}