Skip to main content

datumesh_http/
publish.rs

1//! Publishing stored cells as the mesh (`datumesh-mesh publish-stored`): a JSON lines file of
2//! stored functions and their cells, signed with the mesh's key, put in the store.
3//!
4//! Each line is one of:
5//!
6//! * a function: `{"function": "<ns>/<name>", "signature": {...}, "bound": "<instant>"}`: the
7//!   descriptor `{"code": {"body": "", "lang": "stored"}, "inputs": [], "signature": ...}`
8//!   is put and the name bound to it from `bound` (unless it already is);
9//! * a formula function: `{"formula": "<ns>/<name>", "source": "<formula>", "signature":
10//!   {...}, "bound": "<instant>"}`: defined with `datumesh_formula::definition::define_function`
11//!   (its callees as bound at `bound`) and published (`put_descriptor`, `bind_name` from
12//!   `bound`) unless the name is already bound to that function; its cells are evaluated on
13//!   demand by the mesh;
14//! * a class (since 0.2.2, spec-0.5.0, 16.2): `{"class": "<ns>/<name>", "descriptor": {...as
15//!   written...}, "bound": "<instant>"}`: defined with the formula crate's `define_class` (core's
16//!   rules: the class is under one of the mesh's own namespaces, else `E_CLASS_OWNER`; the method
17//!   functions are those bound as of `bound`), `put_class`, and `bind_class` from `bound` unless the
18//!   name is already bound to that class hash. Method functions come first in the file (or in an
19//!   earlier one), `bound` later than theirs.
20//! * a cell: `{"cell": {"fn": "<ns>/<name>", "args": {<name>: <typed scalar>}}, "value": <typed
21//!   scalar> or {"blob": "<file>", "media_type": "<type>"}, "tx", "asof", "issued"?, "licence",
22//!   "summary"}`: a blob's file (relative to the JSON lines file) is put with `put_blob` and the
23//!   value is `<sha256>:<length>:<media type>`; the receipt is signed with the mesh's key
24//!   (`computed_by` the mesh host, `publisher` the cell's namespace, `valid` its valid-time
25//!   arguments; `issued` defaults to the later of `asof` and the key's start) and stored with
26//!   `put_receipt` and `put_version`.
27//!
28//! The function of a cell is the one its name is bound to as of the cell's `tx`. Re-running a
29//! file changes nothing: descriptors and blobs are content-addressed, signing is deterministic
30//! (the same receipt, the same hash), and a version already there is skipped. The key is never
31//! logged.
32
33use std::io::BufRead;
34use std::path::Path;
35
36use datumesh_core::json::{self, Value};
37use datumesh_core::{Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, ReceiptBuilder, Scalar, SigningKey, Type};
38use datumesh_formula::definition::ClassCallee;
39use datumesh_store::Store;
40
41/// What a run put.
42#[non_exhaustive]
43#[derive(Clone, Debug, Default, PartialEq, Eq)]
44pub struct Published {
45    /// Functions newly bound (stored and formula).
46    pub functions: usize,
47    /// Classes newly bound (since 0.2.2).
48    pub classes: usize,
49    /// Cell lines read.
50    pub cells: usize,
51    /// Receipts stored (new or already there).
52    pub receipts: usize,
53    /// Versions newly added.
54    pub versions: usize,
55    /// Blobs put (new or already there).
56    pub blobs: usize,
57    /// Blob bytes put.
58    pub blob_bytes: u64,
59}
60
61/// Why a run stopped: the line and what was wrong.
62#[derive(Clone, Debug, PartialEq, Eq)]
63pub struct PublishError {
64    /// The line number (1-based; 0 for the file).
65    pub line: usize,
66    /// Why.
67    pub detail: String,
68}
69
70impl std::fmt::Display for PublishError {
71    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72        write!(f, "line {}: {}", self.line, self.detail)
73    }
74}
75
76impl std::error::Error for PublishError {}
77
78fn s(v: &str) -> Value {
79    Value::String(v.to_string())
80}
81
82/// A store as the callees of a definition: a name's binding as of an instant, with its
83/// descriptor; a class name's binding (since 0.2.2); the classes and functions known by hash.
84struct StoreCallees<'a, S: Store + ?Sized>(&'a S);
85
86impl<S: Store + ?Sized> datumesh_formula::definition::Callees for StoreCallees<'_, S> {
87    fn callee(&self, name: &str, at: &str) -> Result<Option<datumesh_formula::definition::Callee>, String> {
88        let Some(b) = self.0.resolve_binding(name, at).map_err(|e| e.to_string())? else { return Ok(None) };
89        let bytes = self.0.descriptor(&b.fn_hash).ok_or_else(|| format!("descriptor {} is not stored", b.fn_hash))?;
90        let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
91        Ok(Some(datumesh_formula::definition::Callee { fn_hash: b.fn_hash, start: b.tx, descriptor }))
92    }
93
94    fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
95        let Some(b) = self.0.resolve_class_binding(name, at).map_err(|e| e.to_string())? else { return Ok(None) };
96        let bytes = self
97            .0
98            .class(&b.class_hash)
99            .map_err(|e| e.to_string())?
100            .ok_or_else(|| format!("class {} is not stored", b.class_hash))?;
101        let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
102        Ok(Some(ClassCallee { class_hash: b.class_hash, start: b.tx, descriptor }))
103    }
104
105    fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
106        let Some(bytes) = self.0.class(class_hash).map_err(|e| e.to_string())? else { return Ok(None) };
107        ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
108    }
109
110    fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
111        let Some(bytes) = self.0.descriptor(fn_hash) else { return Ok(None) };
112        json::parse(&bytes).map(Some).map_err(|e| e.to_string())
113    }
114}
115
116/// The receipt JSON of a signed receipt: its payload's members and `sig`.
117pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
118    let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
119    if let Ok(Value::Object(m)) = json::parse(r.payload()) {
120        members.extend(m);
121    }
122    Value::Object(members).to_jcs()
123}
124
125/// Publishes the file at `path` into `store` as the mesh `host` signing with `key` (see the
126/// module documentation). The mesh's own namespaces are `host` alone: see [`publish_stored_in`].
127pub fn publish_stored<S: Store + ?Sized>(
128    store: &S,
129    key: &SigningKey,
130    host: &str,
131    path: &Path,
132) -> Result<Published, PublishError> {
133    publish_stored_in(store, key, host, &[host.to_string()], path)
134}
135
136/// As [`publish_stored`], for a mesh that owns the namespaces (hosts) `owned` (`Config::namespaces`):
137/// a class line is accepted for a class under one of them, and refused with `E_CLASS_OWNER` (in the
138/// [`PublishError::detail`]) under any other. Since 0.2.2.
139pub fn publish_stored_in<S: Store + ?Sized>(
140    store: &S,
141    key: &SigningKey,
142    host: &str,
143    owned: &[String],
144    path: &Path,
145) -> Result<Published, PublishError> {
146    let file =
147        std::fs::File::open(path).map_err(|e| PublishError { line: 0, detail: format!("{}: {e}", path.display()) })?;
148    let base = path.parent().unwrap_or(Path::new("."));
149    let mut out = Published::default();
150    // The key's start: the earliest `issued`.
151    let key_from = store
152        .namespace(host)
153        .ok()
154        .flatten()
155        .and_then(|n| json::parse(&n.keydoc).ok())
156        .and_then(|d| match d.get("keys") {
157            Some(Value::Array(ks)) => ks
158                .iter()
159                .find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
160                .and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
161            _ => None,
162        })
163        .ok_or_else(|| PublishError {
164            line: 0,
165            detail: format!("the key {} is not in {host}'s key document in the store", key.key_id()),
166        })?;
167    for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
168        let n = i + 1;
169        let err = |d: String| PublishError { line: n, detail: d };
170        let line = line.map_err(|e| err(e.to_string()))?;
171        if line.trim().is_empty() {
172            continue;
173        }
174        let v = json::parse(line.as_bytes()).map_err(|e| err(e.to_string()))?;
175        if let Some(name) = v.get("class").and_then(Value::as_str) {
176            let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
177            let bound = v.get("bound").and_then(Value::as_str).ok_or_else(|| err("a class without bound".into()))?;
178            // 16.2 step 1: the class is under one of the mesh's own namespaces. Core checks that the
179            // class's namespace is the definer's; a host the mesh does not own is defined by the mesh's
180            // host, which no class name under it has, so core answers E_CLASS_OWNER.
181            let host_of_name = name.split('/').next().unwrap_or("");
182            let definer = if owned.iter().any(|o| o == host_of_name) {
183                name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
184            } else {
185                host
186            };
187            let def = datumesh_formula::definition::define_class(
188                name,
189                &descriptor.to_jcs(),
190                definer,
191                &StoreCallees(store),
192                bound,
193            )
194            .map_err(|e| err(format!("define {name}: {e}")))?;
195            let bound_hash = store.put_class(&def.bytes).map_err(|e| err(e.to_string()))?;
196            match store.resolve_class(name, "9999-12-31T23:59:59.999999Z").map_err(|e| err(e.to_string()))? {
197                Some(h) if h == bound_hash => {}
198                _ => {
199                    store.bind_class(name, &bound_hash, bound).map_err(|e| err(format!("bind {name}: {e}")))?;
200                    out.classes += 1;
201                }
202            }
203            continue;
204        }
205        if let Some(name) = v.get("function").and_then(Value::as_str) {
206            let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
207            let bound = v.get("bound").and_then(Value::as_str).ok_or_else(|| err("a function without bound".into()))?;
208            let d = Value::Object(vec![
209                ("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
210                ("inputs".into(), Value::Array(vec![])),
211                ("signature".into(), sig.clone()),
212            ]);
213            let fd = FunctionDescriptor::from_value(&d).map_err(|e| err(e.to_string()))?;
214            let h = store.put_descriptor(&fd.canonical()).map_err(|e| err(e.to_string()))?;
215            match store.resolve_binding(name, "9999-12-31T23:59:59.999999Z").map_err(|e| err(e.to_string()))? {
216                Some(b) if b.fn_hash == h => {}
217                _ => {
218                    store.bind_name(name, &h, bound).map_err(|e| err(format!("bind {name}: {e}")))?;
219                    out.functions += 1;
220                }
221            }
222            continue;
223        }
224        if let Some(name) = v.get("formula").and_then(Value::as_str) {
225            let source =
226                v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
227            let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
228            let bound = v.get("bound").and_then(Value::as_str).ok_or_else(|| err("a formula without bound".into()))?;
229            let def = datumesh_formula::definition::define_function(
230                name,
231                source,
232                sig,
233                &Default::default(),
234                &StoreCallees(store),
235                bound,
236            )
237            .map_err(|e| err(format!("define {name}: {e}")))?;
238            let h = store.put_descriptor(&def.bytes).map_err(|e| err(e.to_string()))?;
239            match store.resolve_binding(name, "9999-12-31T23:59:59.999999Z").map_err(|e| err(e.to_string()))? {
240                Some(b) if b.fn_hash == h => {}
241                _ => {
242                    store.bind_name(name, &h, bound).map_err(|e| err(format!("bind {name}: {e}")))?;
243                    out.functions += 1;
244                }
245            }
246            continue;
247        }
248        let cell_v = v.get("cell").ok_or_else(|| err("neither a function nor a cell".into()))?;
249        let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
250        let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
251        let field =
252            |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string).ok_or_else(|| err(format!("no {k}")));
253        let (tx, asof, licence, summary) = (field("tx")?, field("asof")?, field("licence")?, field("summary")?);
254        let issued = v
255            .get("issued")
256            .and_then(Value::as_str)
257            .map(str::to_string)
258            .unwrap_or_else(|| std::cmp::max(asof.clone(), key_from.clone()));
259        let fn_hash = store
260            .resolve_name(name, &tx)
261            .map_err(|e| err(e.to_string()))?
262            .ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
263        let d = store.descriptor(&fn_hash).ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
264        let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
265        let mut args = Args::new();
266        for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
267            args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
268        }
269        let valid: Args = args
270            .iter()
271            .filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
272            .map(|(k, a)| (k.clone(), a.clone()))
273            .collect();
274        let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
275        let value = match value_v.get("blob").and_then(Value::as_str) {
276            Some(file) => {
277                let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
278                let bytes = std::fs::read(base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
279                let sha = store.put_blob(&bytes).map_err(|e| err(e.to_string()))?;
280                out.blobs += 1;
281                out.blob_bytes += bytes.len() as u64;
282                Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
283                    .map_err(|e| err(e.to_string()))?
284            }
285            None => Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?,
286        };
287        let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
288        let r = ReceiptBuilder::new()
289            .cell(cell.clone())
290            .value(value)
291            .valid(valid)
292            .asof(asof.as_str())
293            .tx(tx.as_str())
294            .issued(issued.as_str())
295            .publisher(ns)
296            .signer_host(host)
297            .licence(licence.as_str())
298            .summary(summary.as_str())
299            .sign(key)
300            .map_err(|e| err(e.to_string()))?;
301        let h = store.put_receipt(&receipt_json(&r)).map_err(|e| err(e.to_string()))?;
302        out.cells += 1;
303        out.receipts += 1;
304        let id = cell.id();
305        match store.latest(&id).map_err(|e| err(e.to_string()))? {
306            Some(l) if l.tx.as_str() >= tx.as_str() => {
307                if store.version_asof(&id, &tx).ok().flatten().is_none_or(|v| v.receipt_hash != h) {
308                    return Err(err(format!("{id} has a version at or after {tx} that is not this receipt")));
309                }
310            }
311            _ => {
312                store.put_version(&id, &tx, &h).map_err(|e| err(e.to_string()))?;
313                out.versions += 1;
314            }
315        }
316    }
317    Ok(out)
318}