use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::sync::Arc;
use datumesh_core::{
inherit_licence, lexical, Args, Cell, Declarations, Error as CoreError, ErrorCode, Fault, FunctionDescriptor,
Inheritance, KeyDocument, LicenceReason, Manifest, Receipt, ReceiptBuilder, Scalar, Slice,
};
use crate::cache::Parsed;
use crate::check::{check_args, valid_args};
use crate::engine::{Engine, STORED};
use crate::error::{EngineError, EvalError, StoreError};
use crate::memo::Entry;
use crate::runtime::{Budget, FnBinding, Input, Inputs, Member, SlicePattern};
use crate::sign::receipt_json;
use crate::slice::{cells_slice, extent_cells, spec_of, versions_slice, Spec};
pub(crate) const MIN_INSTANT: &str = "0001-01-01T00:00:00.000000Z";
pub(crate) const MAX_INSTANT: &str = "9999-12-31T23:59:59.999999Z";
pub(crate) fn just_after(a: &str) -> String {
format!("{a}~")
}
pub(crate) type Names = BTreeMap<(String, Option<String>), String>;
#[derive(Debug)]
pub(crate) struct Ans {
pub(crate) id: String,
pub(crate) hash: String,
pub(crate) bytes: Arc<Vec<u8>>,
pub(crate) value: Scalar,
pub(crate) tx: String,
pub(crate) asof: String,
pub(crate) licence: String,
pub(crate) lo: String,
pub(crate) hi: Option<String>,
pub(crate) names: Arc<Names>,
pub(crate) cell: Cell,
pub(crate) inputs: Vec<Use>,
pub(crate) memo: bool,
}
#[derive(Clone, Debug)]
pub(crate) enum Use {
Cell { id: String, cell: Cell, hash: String, value: Scalar, at: Option<String> },
Slice(Arc<SliceRead>),
}
fn input_deps(inputs: &[Use]) -> impl Iterator<Item = &String> {
inputs.iter().flat_map(|u| -> Box<dyn Iterator<Item = &String> + '_> {
match u {
Use::Cell { id, .. } => Box::new(std::iter::once(id)),
Use::Slice(s) if s.computed => Box::new(s.members.iter().map(|m| &m.id)),
Use::Slice(_) => Box::new(std::iter::empty()),
}
})
}
fn input_patterns(inputs: &[Use]) -> impl Iterator<Item = &Arc<Slice>> {
inputs.iter().filter_map(|u| match u {
Use::Slice(s) if !s.computed => Some(&s.slice),
_ => None,
})
}
impl Ans {
pub(crate) fn deps(&self) -> impl Iterator<Item = &String> {
input_deps(&self.inputs)
}
pub(crate) fn patterns(&self) -> impl Iterator<Item = &Arc<Slice>> {
input_patterns(&self.inputs)
}
}
#[derive(Debug)]
pub(crate) struct FaultAns {
pub(crate) id: String,
pub(crate) fault: Fault,
pub(crate) known: String,
pub(crate) lo: String,
pub(crate) hi: Option<String>,
pub(crate) names: Arc<Names>,
pub(crate) inputs: Vec<Use>,
pub(crate) faulted: Vec<String>,
pub(crate) memo: bool,
}
impl FaultAns {
pub(crate) fn deps(&self) -> impl Iterator<Item = &String> {
self.faulted.iter().chain(input_deps(&self.inputs))
}
pub(crate) fn patterns(&self) -> impl Iterator<Item = &Arc<Slice>> {
input_patterns(&self.inputs)
}
}
#[derive(Debug)]
pub(crate) struct SliceRead {
pub(crate) entry: String,
pub(crate) manifest: Arc<Vec<u8>>,
pub(crate) slice: Arc<Slice>,
pub(crate) fn_name: String,
pub(crate) fn_hash: String,
pub(crate) computed: bool,
pub(crate) members: Vec<MemberRec>,
member_ans: Vec<Arc<Ans>>,
lo: Option<String>,
hi: Option<String>,
tx: Option<String>,
memo: bool,
}
#[derive(Clone, Debug)]
pub(crate) struct MemberRec {
pub(crate) id: String,
pub(crate) hash: String,
pub(crate) value: Scalar,
args: Args,
tx: String,
asof: String,
}
impl MemberRec {
fn input(&self) -> Input {
Input::new(self.value.clone(), self.hash.clone(), self.tx.clone(), self.asof.clone())
}
}
#[derive(Clone, Debug)]
pub(crate) enum Done {
Value(Arc<Ans>),
NotCovered,
Fault(Arc<FaultAns>),
Unsigned(Scalar, Vec<String>),
}
enum Lookup {
Ready(Done),
Eval(Arc<FunctionDescriptor>),
}
#[derive(Clone, Debug)]
pub(crate) struct Pend {
cell: Cell,
id: String,
desc: Arc<FunctionDescriptor>,
asof: String,
}
#[derive(Clone, Debug)]
enum Rd {
Cell { ans: Arc<Ans>, at: Option<String> },
Slice(Arc<SliceRead>),
}
impl Rd {
fn entry(&self) -> &str {
match self {
Rd::Cell { ans, .. } => &ans.hash,
Rd::Slice(s) => &s.entry,
}
}
}
struct Fresh {
cell_id: String,
tx: String,
hash: String,
bytes: Arc<Vec<u8>>,
computed: bool,
}
enum State {
Start,
Recheck { ans: Arc<Ans>, idx: usize, read: Vec<Rd>, names: Names },
FromStore,
Validate { cand: Arc<Parsed>, idx: usize, read: Vec<Rd>, names: Names },
Eval { cand: Option<Arc<Parsed>> },
}
struct Frame {
cell: Cell,
id: String,
asof: String,
desc: Arc<FunctionDescriptor>,
state: State,
unsigned: bool,
pending: Vec<Pend>,
floor: u64,
}
pub(crate) struct Req<'e> {
eng: &'e Engine,
asof: String,
issued: String,
names: HashMap<String, HashMap<String, Option<(String, String)>>>,
done: HashMap<String, HashMap<String, Done>>,
slices: HashMap<(String, String), Arc<SliceRead>>,
fresh: Vec<Fresh>,
manifests: BTreeMap<String, Arc<Vec<u8>>>,
declarations: HashMap<String, Option<Arc<Declarations>>>,
pub(crate) learned: Vec<Entry>,
budget: Budget,
}
enum Check {
Same(Rd, Option<(String, Option<String>)>, String),
Pending(Vec<Pend>),
Differs,
}
enum SliceOutcome {
Ready(Arc<SliceRead>),
Pending(Vec<Pend>),
Absent(String),
}
enum Step {
Again,
Done(Done),
Push(Vec<Pend>),
}
struct Facts {
tx: Option<String>,
latest: String,
lo: String,
hi: Option<String>,
names: Arc<Names>,
inputs: Vec<Use>,
memo: bool,
entries: BTreeSet<String>,
}
struct How {
dependent: bool,
computed: bool,
memo: bool,
known: Option<String>,
}
fn store_err(op: &'static str) -> impl Fn(datumesh_store::Error) -> EvalError {
move |e| EvalError::Store(StoreError::new(op, e))
}
impl<'e> Req<'e> {
pub(crate) fn new(eng: &'e Engine, asof: &str, issued: &str) -> Req<'e> {
Req {
eng,
asof: asof.to_string(),
issued: issued.to_string(),
names: HashMap::new(),
done: HashMap::new(),
slices: HashMap::new(),
fresh: Vec::new(),
manifests: BTreeMap::new(),
declarations: HashMap::new(),
learned: Vec::new(),
budget: Budget::with_work(eng.fuel, eng.work_limit),
}
}
fn binding(&mut self, fn_name: &str, at: &str) -> Result<Option<(String, String)>, EvalError> {
Ok(self.binding_ref(fn_name, at)?.clone())
}
fn binding_ref(&mut self, fn_name: &str, at: &str) -> Result<&Option<(String, String)>, EvalError> {
let known = self.names.get(at).is_some_and(|m| m.contains_key(fn_name));
if !known {
let b = self
.eng
.store
.resolve_binding(fn_name, at)
.map_err(store_err("resolve_binding"))?
.map(|b| (b.fn_hash, b.tx));
self.names.entry(at.to_string()).or_default().insert(fn_name.to_string(), b);
}
Ok(self.names.get(at).and_then(|m| m.get(fn_name)).unwrap_or(&None))
}
fn resolve(&mut self, fn_name: &str, at: &str) -> Result<Option<String>, EvalError> {
Ok(self.binding_ref(fn_name, at)?.as_ref().map(|(h, _)| h.clone()))
}
fn resolves_to(&mut self, fn_name: &str, at: &str, hash: &str) -> Result<bool, EvalError> {
Ok(self.binding_ref(fn_name, at)?.as_ref().is_some_and(|(h, _)| h == hash))
}
fn names_hold(&mut self, names: &Names, asof: &str) -> Result<bool, EvalError> {
for ((name, at), hash) in names {
if !self.resolves_to(name, at.as_deref().unwrap_or(asof), hash)? {
return Ok(false);
}
}
Ok(true)
}
fn set_done(&mut self, asof: &str, id: &str, d: Done) {
match self.done.get_mut(asof) {
Some(m) => {
m.insert(id.to_string(), d);
}
None => {
self.done.entry(asof.to_string()).or_default().insert(id.to_string(), d);
}
}
}
fn done_at(&self, asof: &str, id: &str) -> Option<&Done> {
self.done.get(asof).and_then(|m| m.get(id))
}
fn lookup(
&mut self,
cell: &Cell,
id: &str,
desc: Arc<FunctionDescriptor>,
asof: &str,
) -> Result<Lookup, EvalError> {
if let Some(d) = self.done_at(asof, id) {
return Ok(Lookup::Ready(d.clone()));
}
if let Some(d) = self.memo_done(id, asof)? {
self.set_done(asof, id, d.clone());
return Ok(Lookup::Ready(d));
}
if desc.lang == STORED {
let d = self.stored(cell, id, asof)?;
self.set_done(asof, id, d.clone());
return Ok(Lookup::Ready(d));
}
Ok(Lookup::Eval(desc))
}
pub(crate) fn memo_hit(&mut self, id: &str, asof: &str) -> Result<Option<Arc<Ans>>, EvalError> {
let Some(a) = self.eng.memo.get(id, asof) else { return Ok(None) };
if !self.names_hold(&a.names, asof)? {
return Ok(None);
}
Ok(Some(a))
}
fn memo_fault_hit(&mut self, id: &str, asof: &str) -> Result<Option<Arc<FaultAns>>, EvalError> {
let Some(f) = self.eng.memo.get_fault(id, asof) else { return Ok(None) };
if !self.names_hold(&f.names, asof)? {
return Ok(None);
}
Ok(Some(f))
}
pub(crate) fn memo_done(&mut self, id: &str, asof: &str) -> Result<Option<Done>, EvalError> {
if let Some(a) = self.memo_hit(id, asof)? {
return Ok(Some(Done::Value(a)));
}
Ok(self.memo_fault_hit(id, asof)?.map(Done::Fault))
}
pub(crate) fn asof(&self) -> String {
self.asof.clone()
}
pub(crate) fn stored(&mut self, cell: &Cell, id: &str, asof: &str) -> Result<Done, EvalError> {
let found = self.eng.store.stored_asof_with_receipt(id, asof).map_err(store_err("stored_asof"))?;
let Some((v, bytes)) = found else { return Ok(Done::NotCovered) };
let p = self.eng.parsed(&v.receipt_hash, &bytes).map_err(EvalError::Store)?;
let hi = if self.eng.memo.enabled() {
let later = self.eng.store.versions(id, &v.tx, MAX_INSTANT).map_err(store_err("versions"))?;
later.into_iter().filter(|w| w.tx > v.tx).map(|w| w.asof).min()
} else {
None
};
let r = &p.receipt;
let a = Arc::new(Ans {
id: id.to_string(),
hash: v.receipt_hash.clone(),
bytes: p.bytes.clone(),
value: r.value().clone(),
tx: r.tx().to_string(),
asof: r.asof().to_string(),
licence: r.licence().to_string(),
lo: r.asof().to_string(),
hi,
names: Arc::new(Names::new()),
cell: cell.clone(),
inputs: Vec::new(),
memo: true,
});
self.learned.push(Entry::Value(a.clone()));
Ok(Done::Value(a))
}
pub(crate) fn run(
&mut self,
cell: Cell,
desc: Arc<FunctionDescriptor>,
unsigned: bool,
) -> Result<Done, EngineError> {
let id = cell.id();
let asof = self.asof.clone();
let state = if unsigned { State::Eval { cand: None } } else { State::Start };
let key = |asof: &str, id: &str| format!("{asof} {id}");
let mut stack = vec![Frame {
cell,
id: id.clone(),
asof: asof.clone(),
desc,
state,
unsigned,
pending: Vec::new(),
floor: 0,
}];
let mut on_stack: HashSet<String> = HashSet::from([key(&asof, &id)]);
while let Some(top) = stack.last_mut() {
if let Some(p) = top.pending.pop() {
if self.done_at(&p.asof, &p.id).is_some() {
continue;
}
if !on_stack.insert(key(&p.asof, &p.id)) {
return Err(EngineError::Cycle { cell_id: p.id });
}
if stack.len() >= self.eng.depth_limit {
return Err(EngineError::DepthExceeded { limit: self.eng.depth_limit });
}
stack.push(Frame {
cell: p.cell,
id: p.id,
asof: p.asof,
desc: p.desc,
state: State::Start,
unsigned: false,
pending: Vec::new(),
floor: 0,
});
continue;
}
let step = match top.state {
State::Start => self.start_frame(top),
State::Recheck { .. } => self.recheck_frame(top)?,
State::FromStore => self.store_frame(top)?,
State::Validate { .. } => self.validate_frame(top)?,
State::Eval { .. } => self.eval_frame(top)?,
};
match step {
Step::Again => {}
Step::Done(d) => {
if let Some(f) = stack.pop() {
on_stack.remove(&key(&f.asof, &f.id));
match &d {
Done::Value(a) => self.learned.push(Entry::Value(a.clone())),
_ => {
if let Done::Fault(fa) = &d {
self.learned.push(Entry::Fault(fa.clone()));
}
if let Some(parent) = stack.last_mut() {
parent.pending.clear();
}
}
}
self.set_done(&f.asof, &f.id, d);
}
}
Step::Push(mut list) => {
list.reverse();
if let Some(top) = stack.last_mut() {
top.pending = list;
}
}
}
}
self.done_at(&asof, &id)
.cloned()
.ok_or_else(|| EngineError::Runtime { cell_id: id, message: "no answer".into() })
}
fn start_frame(&mut self, f: &mut Frame) -> Step {
f.state = match self.eng.memo.stale(&f.id, &f.asof) {
Some(ans) => State::Recheck { ans, idx: 0, read: Vec::new(), names: Names::new() },
None => State::FromStore,
};
Step::Again
}
fn recheck_frame(&mut self, f: &mut Frame) -> Result<Step, EngineError> {
let State::Recheck { ans, idx, read, names } = &mut f.state else { return Ok(Step::Again) };
let ans = ans.clone();
let asof = f.asof.clone();
while let Some(u) = ans.inputs.get(*idx) {
let check = match u {
Use::Cell { id, cell, hash, value, at } => {
self.check_use(cell, id, hash, value, at.as_deref(), &asof, &ans.asof)?
}
Use::Slice(old) => {
let old_members: Vec<(String, Scalar)> =
old.members.iter().map(|m| (m.id.clone(), m.value.clone())).collect();
self.check_slice(&old.entry, &old.slice, Some(&old_members), &asof)?
}
};
match check {
Check::Same(rd, name, hash) => {
read.push(rd);
if let Some(name) = name {
names.insert(name, hash);
}
*idx += 1;
}
Check::Pending(list) => return Ok(Step::Push(list)),
Check::Differs => {
f.state = State::FromStore;
return Ok(Step::Again);
}
}
}
let read = std::mem::take(read);
let names = std::mem::take(names);
let parsed = self.eng.parsed(&ans.hash, &ans.bytes)?;
if !self.reusable(f, &read, &parsed.receipt)? {
f.state = State::FromStore;
return Ok(Step::Again);
}
let facts = self.facts(&read, names);
Ok(Step::Done(Done::Value(self.kept(
f,
facts,
&ans.hash,
&ans.bytes,
&ans.value,
(&ans.tx, &ans.asof),
&ans.licence,
))))
}
#[allow(clippy::too_many_arguments)]
fn kept(
&self,
f: &Frame,
facts: Facts,
hash: &str,
bytes: &Arc<Vec<u8>>,
value: &Scalar,
(tx, asof): (&str, &str),
licence: &str,
) -> Arc<Ans> {
let (lo, hi) = if self.asof_dependent(f) {
(f.asof.clone(), Some(just_after(&f.asof)))
} else {
(facts.lo.as_str().max(asof).to_string(), facts.hi.clone())
};
Arc::new(Ans {
id: f.id.clone(),
hash: hash.to_string(),
bytes: bytes.clone(),
value: value.clone(),
tx: tx.to_string(),
asof: asof.to_string(),
licence: licence.to_string(),
lo,
hi,
names: facts.names,
cell: f.cell.clone(),
inputs: facts.inputs,
memo: facts.memo,
})
}
fn asof_dependent(&self, f: &Frame) -> bool {
self.eng.asof_dependent(&f.cell.fn_hash, &f.desc)
}
fn store_frame(&mut self, f: &mut Frame) -> Result<Step, EngineError> {
let found = self
.eng
.store
.computed_asof_with_receipt(&f.id, &f.asof)
.map_err(|e| StoreError::new("computed_asof", e))?;
f.state = match found {
Some((v, bytes)) => {
let cand = self.eng.parsed(&v.receipt_hash, &bytes)?;
if self.asof_dependent(f) && cand.receipt.asof() != f.asof {
State::Eval { cand: None }
} else {
State::Validate { cand, idx: 0, read: Vec::new(), names: Names::new() }
}
}
None => State::Eval { cand: None },
};
Ok(Step::Again)
}
fn validate_frame(&mut self, f: &mut Frame) -> Result<Step, EngineError> {
let State::Validate { cand, idx, read, names } = &mut f.state else { return Ok(Step::Again) };
let cand = cand.clone();
let asof = f.asof.clone();
while let Some(h) = cand.receipt.inputs().get(*idx) {
let check = if h.starts_with("slice:") {
self.check_entry(h, &asof)?
} else {
self.check_input(h, &asof, cand.receipt.asof())?
};
match check {
Check::Same(rd, name, hash) => {
read.push(rd);
if let Some(name) = name {
names.insert(name, hash);
}
*idx += 1;
}
Check::Pending(list) => return Ok(Step::Push(list)),
Check::Differs => {
f.state = State::Eval { cand: Some(cand) };
return Ok(Step::Again);
}
}
}
let read = std::mem::take(read);
let names = std::mem::take(names);
if !self.reusable(f, &read, &cand.receipt)? {
f.state = State::Eval { cand: Some(cand) };
return Ok(Step::Again);
}
let facts = self.facts(&read, names);
let r = &cand.receipt;
Ok(Step::Done(Done::Value(self.kept(
f,
facts,
r.hash(),
&cand.bytes,
r.value(),
(r.tx(), r.asof()),
r.licence(),
))))
}
fn check_input(&mut self, h: &str, asof: &str, reader: &str) -> Result<Check, EngineError> {
let Some(input) = self.receipt(h)? else { return Ok(Check::Differs) };
let r = &input.receipt;
self.check_use(r.cell(), &r.cell().id(), h, r.value(), None, asof, reader)
}
#[allow(clippy::too_many_arguments)]
fn check_use(
&mut self,
cell: &Cell,
id: &str,
h: &str,
value: &Scalar,
at: Option<&str>,
asof: &str,
reader: &str,
) -> Result<Check, EngineError> {
let when = at.unwrap_or(asof).to_string();
let name = cell.function_name();
let fail = |e: EvalError| engine_error(e, id);
let pinned = match at {
Some(_) => false,
None => {
let current = self.binding_ref(&name, &when).map_err(fail)?.clone();
let at_reader = match current {
None => None,
Some((bound, start)) if start.as_str() <= reader => Some(bound),
Some(_) => self.resolve(&name, reader).map_err(fail)?,
};
at_reader.as_deref() != Some(cell.fn_hash.as_str())
}
};
if !pinned && !self.resolves_to(&name, &when, &cell.fn_hash).map_err(fail)? {
return Ok(Check::Differs);
}
self.budget.work(1).map_err(|e| engine_error(e, id))?;
let Some(desc) = self.eng.descriptor_of(&cell.fn_hash)? else { return Ok(Check::Differs) };
match self.lookup(cell, id, desc, &when).map_err(|e| engine_error(e, id))? {
Lookup::Ready(Done::Value(a)) if a.hash == h || &a.value == value => Ok(Check::Same(
Rd::Cell { ans: a, at: at.map(str::to_string) },
(!pinned).then(|| (name, at.map(str::to_string))),
cell.fn_hash.clone(),
)),
Lookup::Ready(_) => Ok(Check::Differs),
Lookup::Eval(desc) => {
Ok(Check::Pending(vec![Pend { cell: cell.clone(), id: id.to_string(), desc, asof: when }]))
}
}
}
fn check_entry(&mut self, entry: &str, asof: &str) -> Result<Check, EngineError> {
let Some(bytes) = self.eng.store.manifest(entry) else { return Ok(Check::Differs) };
let Ok(m) = Manifest::from_json(&bytes) else { return Ok(Check::Differs) };
let slice = Arc::new(m.slice().clone());
let check = self.check_slice(entry, &slice, None, asof)?;
let Check::Same(Rd::Slice(now), name, hash) = check else { return Ok(check) };
if now.entry == entry {
return Ok(Check::Same(Rd::Slice(now), name, hash));
}
let by_hash: HashMap<&str, &MemberRec> = now.members.iter().map(|x| (x.hash.as_str(), x)).collect();
let mut old: Vec<(String, Scalar)> = Vec::with_capacity(m.members().len());
for h in m.members() {
match by_hash.get(h.as_str()) {
Some(x) => old.push((x.id.clone(), x.value.clone())),
None => match self.receipt(h)? {
Some(p) => old.push((p.receipt.cell().id(), p.receipt.value().clone())),
None => return Ok(Check::Differs),
},
}
}
Ok(if same_members(&old, &now) { Check::Same(Rd::Slice(now), name, hash) } else { Check::Differs })
}
fn check_slice(
&mut self,
entry: &str,
slice: &Arc<Slice>,
old: Option<&[(String, Scalar)]>,
asof: &str,
) -> Result<Check, EngineError> {
let fail = |e: EvalError| engine_error(e, entry);
let name = slice.function_name();
if !self.resolves_to(&name, asof, slice.fn_hash()).map_err(fail)? {
return Ok(Check::Differs);
}
let Some(desc) = self.eng.descriptor_of(slice.fn_hash())? else { return Ok(Check::Differs) };
let Some(spec) = spec_of(slice) else { return Ok(Check::Differs) };
let now = match self.slice_at(&name, slice.clone(), desc, spec, asof).map_err(fail)? {
SliceOutcome::Ready(s) => s,
SliceOutcome::Pending(list) => return Ok(Check::Pending(list)),
SliceOutcome::Absent(_) => return Ok(Check::Differs),
};
self.budget.work(1 + now.members.len() as u64).map_err(fail)?;
let same = match old {
None => true,
Some(o) => now.entry == entry || same_members(o, &now),
};
let fn_hash = slice.fn_hash().to_string();
Ok(if same { Check::Same(Rd::Slice(now), Some((name, None)), fn_hash) } else { Check::Differs })
}
fn slice_at(
&mut self,
fn_name: &str,
slice: Arc<Slice>,
desc: Arc<FunctionDescriptor>,
spec: Spec,
asof: &str,
) -> Result<SliceOutcome, EvalError> {
let key = (asof.to_string(), slice.id().to_string());
if let Some(s) = self.slices.get(&key) {
return Ok(SliceOutcome::Ready(s.clone()));
}
let fn_hash = slice.fn_hash().to_string();
let computed = desc.lang != STORED;
let mut members = Vec::new();
let mut member_ans = Vec::new();
let (lo, hi, memo);
match (&spec, computed) {
(Spec::Cells { pattern }, true) => {
let cells = extent_cells(fn_name, &fn_hash, &desc, pattern).map_err(EvalError::InvalidInput)?;
let mut pending = Vec::new();
let mut absent: Option<String> = None;
for cell in cells {
let id = cell.id();
match self.lookup(&cell, &id, desc.clone(), asof)? {
Lookup::Ready(Done::Value(a)) => member_ans.push(a),
Lookup::Ready(Done::Fault(fa)) => {
absent = Some(fa.known.clone());
break;
}
Lookup::Ready(_) => {
absent = Some(MIN_INSTANT.to_string());
break;
}
Lookup::Eval(d) => pending.push(Pend { cell, id, desc: d, asof: asof.to_string() }),
}
}
if !pending.is_empty() {
return Ok(SliceOutcome::Pending(pending));
}
if let Some(known) = absent {
return Ok(SliceOutcome::Absent(known));
}
for a in &member_ans {
members.push(MemberRec {
id: a.id.clone(),
hash: a.hash.clone(),
value: a.value.clone(),
args: a.cell.args.clone(),
tx: a.tx.clone(),
asof: a.asof.clone(),
});
}
lo = member_ans.iter().map(|a| a.lo.clone()).max();
hi = member_ans.iter().filter_map(|a| a.hi.clone()).min();
memo = member_ans.iter().all(|a| a.memo);
}
(_, true) => {
return Err(EvalError::InvalidInput(CoreError::new(
ErrorCode::Slice,
"$",
"versions() of a computed function: versions are of stored cells",
)))
}
(Spec::Cells { .. } | Spec::Versions { .. }, false) => {
let found = self.eng.store.slice_values(&slice, asof).map_err(store_err("slice_values"))?;
for v in found {
members.push(MemberRec {
id: v.version.cell_id,
hash: v.version.receipt_hash,
value: v.value,
args: v.cell.args,
tx: v.version.tx,
asof: v.version.asof,
});
}
hi = if self.eng.memo.enabled()
&& self.eng.store.slice_changed_after(&slice, asof).map_err(store_err("slice_changed_after"))?
{
Some(just_after(asof))
} else {
None
};
let to = match &spec {
Spec::Versions { to, .. } => Some(to.clone()),
Spec::Cells { .. } => None,
};
lo = members.iter().map(|m| m.asof.clone()).chain(to).max();
memo = true;
}
}
let mut hashes: Vec<String> = members.iter().map(|m| m.hash.clone()).collect();
hashes.sort();
hashes.dedup();
let manifest = Manifest::new((*slice).clone(), hashes).map_err(EvalError::InvalidInput)?;
let s = Arc::new(SliceRead {
entry: manifest.entry(),
manifest: Arc::new(manifest.canonical()),
fn_name: fn_name.to_string(),
fn_hash,
computed,
tx: members.iter().map(|m| m.tx.clone()).max(),
members,
member_ans,
lo,
hi,
memo,
slice,
});
self.slices.insert(key, s.clone());
Ok(SliceOutcome::Ready(s))
}
fn receipt(&self, hash: &str) -> Result<Option<Arc<Parsed>>, EngineError> {
if let Some(p) = self.eng.receipts.get(hash) {
return Ok(Some(p));
}
match self.eng.store.receipt(hash) {
Some(bytes) => Ok(Some(self.eng.parsed(hash, &bytes)?)),
None => Ok(None),
}
}
fn facts(&self, read: &[Rd], own: Names) -> Facts {
fn raise(slot: &mut String, v: &str) {
if v > slot.as_str() {
*slot = v.to_string();
}
}
fn lower(hi: &mut Option<String>, v: &Option<String>) {
if let Some(v) = v {
if hi.as_ref().is_none_or(|h| v < h) {
*hi = Some(v.clone());
}
}
}
let mut tx: Option<String> = None;
let mut latest = MIN_INSTANT.to_string();
let mut lo = MIN_INSTANT.to_string();
let mut hi: Option<String> = None;
let mut names: Option<Arc<Names>> = None;
let add = |names: &mut Option<Arc<Names>>, from: &Arc<Names>| match names {
None => *names = Some(from.clone()),
Some(acc) => {
if from.keys().any(|k| !acc.contains_key(k)) {
let m = Arc::make_mut(acc);
for (k, v) in from.iter() {
m.entry(k.clone()).or_insert_with(|| v.clone());
}
}
}
};
let mut extra = Names::new();
let mut inputs = Vec::new();
let mut memo = true;
let mut entries = BTreeSet::new();
for rd in read {
entries.insert(rd.entry().to_string());
match rd {
Rd::Cell { ans, at } => {
if tx.as_ref().is_none_or(|t| ans.tx > *t) {
tx = Some(ans.tx.clone());
}
memo &= ans.memo;
match at {
None => {
raise(&mut latest, &ans.lo);
raise(&mut lo, &ans.lo);
lower(&mut hi, &ans.hi);
add(&mut names, &ans.names);
}
Some(t) => {
raise(&mut latest, t);
raise(&mut latest, &ans.asof);
for ((n, a), v) in ans.names.iter() {
let at = a.clone().unwrap_or_else(|| t.clone());
extra.entry((n.clone(), Some(at))).or_insert_with(|| v.clone());
}
}
}
inputs.push(Use::Cell {
id: ans.id.clone(),
cell: ans.cell.clone(),
hash: ans.hash.clone(),
value: ans.value.clone(),
at: at.clone(),
});
}
Rd::Slice(s) => {
if let Some(t) = &s.tx {
if tx.as_ref().is_none_or(|x| t > x) {
tx = Some(t.clone());
}
}
if let Some(l) = &s.lo {
raise(&mut latest, l);
raise(&mut lo, l);
}
lower(&mut hi, &s.hi);
memo &= s.memo;
extra.entry((s.fn_name.clone(), None)).or_insert_with(|| s.fn_hash.clone());
for a in &s.member_ans {
add(&mut names, &a.names);
}
inputs.push(Use::Slice(s.clone()));
}
}
}
let mut names = names.unwrap_or_default();
let missing = |m: &Names, from: &Names| from.keys().any(|k| !m.contains_key(k));
if missing(&names, &own) || missing(&names, &extra) {
let m = Arc::make_mut(&mut names);
for (k, v) in own.into_iter().chain(extra) {
m.insert(k, v);
}
}
Facts { tx, latest, lo, hi, names, inputs, memo, entries }
}
fn eval_frame(&mut self, f: &mut Frame) -> Result<Step, EngineError> {
let Some(rt) = self.eng.runtimes.get(&f.desc.lang).cloned() else {
return Err(EngineError::NoRuntime { lang: f.desc.lang.clone(), fn_hash: f.cell.fn_hash.clone() });
};
let declared = self.eng.asof_dependent(&f.cell.fn_hash, &f.desc);
let mut budget = self.budget.share();
budget.begin_attempt(f.floor);
let mut reader = Reader {
budget: self.budget.share(),
asof: f.asof.clone(),
req: self,
desc: &f.desc,
read: BTreeMap::new(),
names: Names::new(),
stop: None,
faulted: false,
fault_known: MIN_INSTANT.to_string(),
compared: MIN_INSTANT.to_string(),
future_seen: false,
at_read: false,
asof_read: false,
fault_deps: Vec::new(),
unmemo: false,
};
let result = rt.evaluate(&f.desc, &f.cell, &mut reader, &mut budget);
let Reader {
read,
names,
stop,
faulted,
fault_known,
compared,
future_seen,
at_read,
asof_read,
fault_deps,
unmemo,
..
} = reader;
let fail = |e: EvalError| engine_error(e, &f.id);
if budget.is_exhausted() {
return Err(exhausted(&budget));
}
match stop {
Some(Stop::Pending(list)) => {
f.floor = budget.suspend_attempt();
return Ok(Step::Push(list));
}
Some(Stop::NotCovered) => {
budget.finish_attempt();
if matches!(result, Err(EvalError::Fault(Fault::Future))) && (future_seen || asof_read) {
let read: Vec<Rd> = read.into_values().collect();
let facts = self.facts(&read, names);
let fa = self.fault_answer(f, facts, &fault_deps, Fault::Future, None, false)?;
return Ok(Step::Done(Done::Fault(fa)));
}
return Ok(Step::Done(Done::NotCovered));
}
Some(Stop::Fatal(e)) => return Err(fail(e)),
None => {}
}
budget.finish_attempt();
let read: Vec<Rd> = read.into_values().collect();
let value = match result {
Ok(v) => v,
Err(EvalError::Fault(code)) => {
let facts = self.facts(&read, names);
let latest = facts.latest.clone().max(fault_known).max(compared);
let memo = !unmemo && !at_read;
let latest = (!(declared || asof_read)).then_some(latest);
let fa = self.fault_answer(f, facts, &fault_deps, code, latest.as_deref(), memo)?;
return Ok(Step::Done(Done::Fault(fa)));
}
Err(EvalError::Suspended | EvalError::InputNotCovered { .. }) => {
return Err(EngineError::Runtime {
cell_id: f.id.clone(),
message: "the runtime returned an input's error for an input it did not read".into(),
})
}
Err(e) => return Err(fail(e)),
};
if value.ty() != f.desc.signature.returns.ty {
return Err(EngineError::Runtime {
cell_id: f.id.clone(),
message: format!(
"returned a {} where the function declares {}",
value.ty().as_str(),
f.desc.signature.returns.ty.as_str()
),
});
}
if f.unsigned {
for rd in &read {
if let Rd::Slice(s) = rd {
self.manifests.insert(s.entry.clone(), s.manifest.clone());
}
}
let entries: BTreeSet<String> = read.iter().map(|r| r.entry().to_string()).collect();
return Ok(Step::Done(Done::Unsigned(value, entries.into_iter().collect())));
}
let cand = match &f.state {
State::Eval { cand } => cand.clone(),
_ => None,
};
let how = How {
dependent: declared || asof_read,
computed: !faulted && !at_read && !(asof_read && !declared),
memo: !faulted,
known: Some(if faulted { fault_known.max(compared) } else { compared }),
};
let ans = self.answer_with(f, cand, value, &read, names, how)?;
Ok(Step::Done(Done::Value(ans)))
}
fn fault_answer(
&mut self,
f: &Frame,
mut facts: Facts,
deps: &[Arc<FaultAns>],
fault: Fault,
latest: Option<&str>,
memo: bool,
) -> Result<Arc<FaultAns>, EngineError> {
let mut faulted = Vec::with_capacity(deps.len());
for d in deps {
facts.memo &= d.memo;
if let Some(h) = &d.hi {
if facts.hi.as_ref().is_none_or(|x| h < x) {
facts.hi = Some(h.clone());
}
}
if d.names.keys().any(|k| !facts.names.contains_key(k)) {
let m = Arc::make_mut(&mut facts.names);
for (k, v) in d.names.iter() {
m.entry(k.clone()).or_insert_with(|| v.clone());
}
}
faulted.push(d.id.clone());
}
let (known, lo, hi) = match latest {
None => (f.asof.clone(), f.asof.clone(), Some(just_after(&f.asof))),
Some(latest) => {
let tx = facts.tx.clone().unwrap_or_else(|| MIN_INSTANT.to_string());
let known = self
.binding_asof(latest.to_string(), &facts.names, &f.asof)
.map_err(|e| engine_error(e, &f.id))?
.max(tx);
(known.clone(), known, facts.hi.clone())
}
};
Ok(Arc::new(FaultAns {
id: f.id.clone(),
fault,
known,
lo,
hi,
names: facts.names,
inputs: facts.inputs,
faulted,
memo: memo && facts.memo,
}))
}
fn answer_with(
&mut self,
f: &Frame,
cand: Option<Arc<Parsed>>,
value: Scalar,
read: &[Rd],
own: Names,
how: How,
) -> Result<Arc<Ans>, EngineError> {
let mut facts = self.facts(read, own);
if let Some(k) = &how.known {
if *k > facts.latest {
facts.latest = k.clone();
}
}
let tx = facts.tx.clone().unwrap_or_else(|| MIN_INSTANT.to_string());
let asof = if how.dependent {
f.asof.clone()
} else {
let a =
self.binding_asof(facts.latest.clone(), &facts.names, &f.asof).map_err(|e| engine_error(e, &f.id))?;
a.max(tx.clone())
};
let (lo, hi) = if how.dependent {
(f.asof.clone(), Some(just_after(&f.asof)))
} else {
(facts.lo.as_str().max(asof.as_str()).to_string(), facts.hi.clone())
};
let memo = how.memo && facts.memo;
let licences = self.licences(read)?;
let mut existing = None;
if let Some(c) = &cand {
if c.receipt.value() == &value
&& (!how.dependent || c.receipt.asof() == f.asof)
&& self.entitled_with(f, &licences, &c.receipt)?
{
existing = Some(c.clone());
} else if c.receipt.tx() == tx && c.receipt.asof() == asof {
existing = self.signed_before(f, &facts.entries, &value, &licences)?;
}
}
if let Some(p) = existing {
let r = &p.receipt;
return Ok(Arc::new(Ans {
id: f.id.clone(),
hash: r.hash().to_string(),
bytes: p.bytes.clone(),
value,
tx: r.tx().to_string(),
asof: r.asof().to_string(),
licence: r.licence().to_string(),
lo: lo.as_str().max(r.asof()).to_string(),
hi,
names: facts.names,
cell: f.cell.clone(),
inputs: facts.inputs,
memo,
}));
}
let issued = self.issued.clone();
let rule = self.inherit(f, &licences, &issued, &self.eng.licence.clone())?;
let Some(licence) = rule.licence else {
return Err(EngineError::Unlicensed { cell_id: f.id.clone(), reason: rule.reason });
};
let summary = match &self.eng.summary {
Some(s) => s(&f.cell),
None => format!("{} computed by {}.", f.cell.function_name(), self.eng.host),
};
let receipt = ReceiptBuilder::new()
.cell(f.cell.clone())
.value(value.clone())
.valid(valid_args(&f.cell, &f.desc))
.asof(asof.clone())
.tx(tx.clone())
.issued(self.issued.clone())
.publisher(f.cell.ns.clone())
.signer_host(self.eng.host.clone())
.inputs(facts.entries.iter().cloned())
.licence(licence.clone())
.summary(summary)
.sign(&self.eng.key)
.map_err(EngineError::Sign)?;
let bytes = Arc::new(receipt_json(&receipt));
let hash = receipt.hash().to_string();
for rd in read {
if let Rd::Slice(s) = rd {
self.manifests.insert(s.entry.clone(), s.manifest.clone());
}
}
self.fresh.push(Fresh {
cell_id: f.id.clone(),
tx: tx.clone(),
hash: hash.clone(),
bytes: bytes.clone(),
computed: how.computed,
});
Ok(Arc::new(Ans {
id: f.id.clone(),
hash,
bytes,
value,
tx,
asof,
licence,
lo,
hi,
names: facts.names,
cell: f.cell.clone(),
inputs: facts.inputs,
memo,
}))
}
fn binding_asof(&mut self, latest: String, names: &Names, asof: &str) -> Result<String, EvalError> {
let mut out = latest;
for ((name, at), hash) in names {
match self.binding(name, at.as_deref().unwrap_or(asof))? {
Some((h, start)) if h == *hash => {
if start > out {
out = start;
}
}
_ => return Ok(asof.to_string()),
}
}
Ok(out)
}
fn signed_before(
&mut self,
f: &Frame,
entries: &BTreeSet<String>,
value: &Scalar,
licences: &[String],
) -> Result<Option<Arc<Parsed>>, EngineError> {
let id = f.id.as_str();
let Some(first) = entries.iter().next() else { return Ok(None) };
let candidates = if first.starts_with("slice:") {
self.eng.store.slice_dependents(first).map_err(|e| StoreError::new("slice_dependents", e))?
} else {
self.eng.store.dependents(first).map_err(|e| StoreError::new("dependents", e))?
};
for h in candidates {
let Some(p) = self.receipt(&h)? else { continue };
let r = &p.receipt;
if r.signer_host() == self.eng.host
&& r.value() == value
&& r.inputs().len() == entries.len()
&& r.inputs().iter().zip(entries.iter()).all(|(a, b)| a == b)
&& r.cell().id() == id
&& self.entitled_with(f, licences, r)?
{
return Ok(Some(p));
}
}
Ok(None)
}
fn licences(&mut self, read: &[Rd]) -> Result<Vec<String>, EngineError> {
let mut out = Vec::new();
for rd in read {
match rd {
Rd::Cell { ans, .. } => out.push(ans.licence.clone()),
Rd::Slice(s) if s.computed => out.extend(s.member_ans.iter().map(|a| a.licence.clone())),
Rd::Slice(s) => {
for m in &s.members {
let Some(p) = self.receipt(&m.hash)? else {
return Err(StoreError::new(
"receipt",
datumesh_store::Error::NotFound { what: "receipt", key: m.hash.clone() },
)
.into());
};
out.push(p.receipt.licence().to_string());
}
}
}
}
Ok(out)
}
fn declarations(&mut self, ns: &str) -> Result<Option<Arc<Declarations>>, EngineError> {
let host = ns.split('/').next().unwrap_or(ns);
if let Some(d) = self.declarations.get(host) {
return Ok(d.clone());
}
let d = match self.eng.store.namespace(host).map_err(|e| StoreError::new("namespace", e))? {
Some(n) => KeyDocument::from_json(host, &n.keydoc).ok().and_then(|k| k.declarations).map(Arc::new),
None => None,
};
self.declarations.insert(host.to_string(), d.clone());
Ok(d)
}
fn inherit(
&mut self,
f: &Frame,
licences: &[String],
issued: &str,
mesh_licence: &str,
) -> Result<Inheritance, EngineError> {
let function = f.cell.function_name();
let without = inherit_licence(licences, None, &function, issued, mesh_licence);
if !matches!(without.reason, LicenceReason::NotDeclared | LicenceReason::DeclarationNotOpen) {
return Ok(without);
}
let d = self.declarations(&f.cell.ns)?;
Ok(inherit_licence(licences, d.as_deref(), &function, issued, mesh_licence))
}
fn entitled_with(&mut self, f: &Frame, licences: &[String], r: &Receipt) -> Result<bool, EngineError> {
let then = self.inherit(f, licences, r.issued(), r.licence())?;
if then.licence.as_deref() == Some(r.licence()) {
return Ok(true);
}
let issued = self.issued.clone();
let now = self.inherit(f, licences, &issued, &self.eng.licence.clone())?;
Ok(now.licence.as_deref() == Some(r.licence()))
}
fn reusable(&mut self, f: &Frame, read: &[Rd], r: &Receipt) -> Result<bool, EngineError> {
if r.inputs().is_empty() {
return Ok(true);
}
let licences = self.licences(read)?;
self.entitled_with(f, &licences, r)
}
pub(crate) fn flush(&mut self) -> Result<(), EngineError> {
if !self.fresh.is_empty() {
if let Some(ns) = self.eng.store.namespace(&self.eng.host).map_err(|e| StoreError::new("namespace", e))? {
if let Some(since) = ns.suspended_at {
return Err(EngineError::NamespaceSuspended { host: self.eng.host.clone(), suspended_at: since });
}
}
}
for m in self.manifests.values() {
self.eng.store.put_manifest(m).map_err(|e| StoreError::new("put_manifest", e))?;
}
for f in &self.fresh {
self.eng.store.put_receipt(&f.bytes).map_err(|e| StoreError::new("put_receipt", e))?;
}
for f in self.fresh.iter().filter(|f| f.computed) {
match self.eng.store.put_computed(&f.cell_id, &f.tx, &f.hash) {
Ok(()) => {}
Err(datumesh_store::Error::Conflict { .. }) => {
tracing::warn!(target: "datumesh_engine", cell_id = %f.cell_id, receipt = %f.hash,
"another computed receipt is recorded for this cell at the same tx and asof; kept that one");
}
Err(e) => return Err(StoreError::new("put_computed", e).into()),
}
}
self.fresh.clear();
self.manifests.clear();
Ok(())
}
}
fn same_members(old: &[(String, Scalar)], now: &SliceRead) -> bool {
if old.len() != now.members.len() {
return false;
}
let mut a: Vec<(&str, &Scalar)> = old.iter().map(|(i, v)| (i.as_str(), v)).collect();
let mut b: Vec<(&str, &Scalar)> = now.members.iter().map(|m| (m.id.as_str(), &m.value)).collect();
a.sort_by(|x, y| x.0.cmp(y.0));
b.sort_by(|x, y| x.0.cmp(y.0));
a == b
}
enum Stop {
Pending(Vec<Pend>),
NotCovered,
Fatal(EvalError),
}
struct Reader<'r, 'e> {
req: &'r mut Req<'e>,
asof: String,
desc: &'r FunctionDescriptor,
budget: Budget,
read: BTreeMap<(String, Option<String>), Rd>,
names: Names,
stop: Option<Stop>,
faulted: bool,
fault_known: String,
compared: String,
future_seen: bool,
at_read: bool,
asof_read: bool,
fault_deps: Vec<Arc<FaultAns>>,
unmemo: bool,
}
impl Reader<'_, '_> {
fn fault(&mut self, known: &str, dep: Option<Arc<FaultAns>>) {
self.faulted = true;
if known > self.fault_known.as_str() {
self.fault_known = known.to_string();
}
match dep {
Some(d) => self.fault_deps.push(d),
None => self.unmemo = true,
}
}
fn future(&mut self) -> EvalError {
self.faulted = true;
self.asof_read = true;
self.future_seen = true;
EvalError::Fault(Fault::Future)
}
fn declared(&self, fn_name: &str) -> Result<(), EvalError> {
if !self.desc.inputs.iter().any(|i| i == fn_name) || !fn_name.contains('/') {
return Err(EvalError::UndeclaredInput { fn_name: fn_name.to_string() });
}
Ok(())
}
fn descriptor_for(&self, fn_hash: &str) -> Result<Arc<FunctionDescriptor>, EvalError> {
match self.req.eng.descriptor_of(fn_hash) {
Ok(Some(d)) => Ok(d),
Ok(None) => Err(EvalError::Store(StoreError::new(
"descriptor",
datumesh_store::Error::NotFound { what: "descriptor", key: fn_hash.to_string() },
))),
Err(EngineError::Store(e)) => Err(EvalError::Store(e)),
Err(EngineError::Invalid(e)) => Err(EvalError::InvalidInput(e)),
Err(e) => Err(EvalError::Runtime(e.to_string())),
}
}
fn function(
&mut self,
fn_name: &str,
at: &str,
key_at: Option<&str>,
) -> Result<(String, Arc<FunctionDescriptor>), EvalError> {
let Some(fn_hash) = self.req.resolve(fn_name, at)? else {
return Err(EvalError::NameUnresolved { fn_name: fn_name.to_string(), asof: at.to_string() });
};
self.names.insert((fn_name.to_string(), key_at.map(str::to_string)), fn_hash.clone());
let desc = self.descriptor_for(&fn_hash)?;
Ok((fn_hash, desc))
}
fn read_at(&mut self, fn_name: &str, args: &Args, at: Option<&str>) -> Result<Input, EvalError> {
self.budget.charge_shared(1)?;
self.declared(fn_name)?;
let when = at.unwrap_or(&self.asof).to_string();
let (fn_hash, desc) = self.function(fn_name, &when, at)?;
let cell = cell_named(fn_name, fn_hash, args);
self.cell_read(cell, desc, at)
}
fn read_pinned_at(&mut self, fn_name: &str, fn_hash: &str, args: &Args) -> Result<Input, EvalError> {
self.budget.charge_shared(1)?;
self.declared(fn_name)?;
let desc = self.descriptor_for(fn_hash)?;
let cell = cell_named(fn_name, fn_hash.to_string(), args);
self.cell_read(cell, desc, None)
}
fn cell_read(&mut self, cell: Cell, desc: Arc<FunctionDescriptor>, at: Option<&str>) -> Result<Input, EvalError> {
let when = at.unwrap_or(&self.asof).to_string();
if let Some((k, _)) = cell.args.iter().find(|(_, a)| a.scalar().is_none()) {
return Err(EvalError::InvalidInput(CoreError::new(
ErrorCode::Slice,
format!("$.args.{k}"),
"an input is a cell, not a slice",
)));
}
check_args(&cell, &desc).map_err(EvalError::InvalidInput)?;
let id = cell.id();
if at.is_some() {
self.at_read = true;
}
match self.req.lookup(&cell, &id, desc, &when)? {
Lookup::Ready(Done::Value(a)) => {
let input = Input::new(a.value.clone(), a.hash.clone(), a.tx.clone(), a.asof.clone());
let at = at.map(str::to_string);
self.read.insert((a.hash.clone(), at.clone()), Rd::Cell { ans: a, at });
Ok(input)
}
Lookup::Ready(Done::Fault(fa)) => {
let known = fa.known.clone();
self.fault(&known, Some(fa.clone()));
Err(EvalError::Fault(fa.fault))
}
Lookup::Ready(Done::NotCovered | Done::Unsigned(..)) => {
self.stop = Some(Stop::NotCovered);
Err(EvalError::InputNotCovered { cell_id: id })
}
Lookup::Eval(desc) => {
self.stop = Some(Stop::Pending(vec![Pend { cell, id, desc, asof: when }]));
Err(EvalError::Suspended)
}
}
}
fn slice_read(
&mut self,
fn_name: &str,
slice: Arc<Slice>,
desc: Arc<FunctionDescriptor>,
spec: Spec,
) -> Result<Arc<SliceRead>, EvalError> {
let asof = self.asof.clone();
match self.req.slice_at(fn_name, slice, desc, spec, &asof)? {
SliceOutcome::Ready(s) => {
self.budget.charge_shared(1 + s.members.len() as u64)?;
self.read.insert((s.entry.clone(), None), Rd::Slice(s.clone()));
Ok(s)
}
SliceOutcome::Pending(list) => {
self.stop = Some(Stop::Pending(list));
Err(EvalError::Suspended)
}
SliceOutcome::Absent(known) => {
self.fault(&known, None);
Err(EvalError::Fault(Fault::Absent))
}
}
}
fn guard<T>(&mut self, r: Result<T, EvalError>) -> Result<T, EvalError> {
if let Err(e) = &r {
if !matches!(e, EvalError::Suspended | EvalError::InputNotCovered { .. } | EvalError::Fault(_)) {
self.stop = Some(Stop::Fatal(e.clone()));
}
}
r
}
fn stopped(&self) -> Option<EvalError> {
match &self.stop {
Some(Stop::Pending(..)) => Some(EvalError::Suspended),
Some(Stop::NotCovered) => Some(EvalError::InputNotCovered { cell_id: String::new() }),
Some(Stop::Fatal(e)) => Some(e.clone()),
None => None,
}
}
}
fn instant_arg(s: &str, what: &str) -> Result<(), EvalError> {
lexical::check_instant(s, what).map_err(EvalError::InvalidInput)
}
impl Inputs for Reader<'_, '_> {
fn read(&mut self, fn_name: &str, args: &Args) -> Result<Scalar, EvalError> {
self.read_cell(fn_name, args).map(|i| i.value)
}
fn read_cell(&mut self, fn_name: &str, args: &Args) -> Result<Input, EvalError> {
if let Some(e) = self.stopped() {
return Err(e);
}
let r = self.read_at(fn_name, args, None);
self.guard(r)
}
fn read_asof(&mut self, fn_name: &str, args: &Args, t: &str) -> Result<Input, EvalError> {
if let Some(e) = self.stopped().filter(|_| is_instant(t) && t <= self.asof.as_str()) {
return Err(e);
}
let r = (|| {
instant_arg(t, "$.t")?;
if t > self.asof.as_str() {
return Err(self.future());
}
self.read_at(fn_name, args, Some(t))
})();
self.guard(r)
}
fn read_pinned(&mut self, fn_name: &str, fn_hash: &str, args: &Args) -> Result<Input, EvalError> {
if let Some(e) = self.stopped() {
return Err(e);
}
let r = self.read_pinned_at(fn_name, fn_hash, args);
self.guard(r)
}
fn read_slice(&mut self, fn_name: &str, pattern: &SlicePattern) -> Result<Vec<Member>, EvalError> {
if let Some(e) = self.stopped() {
return Err(e);
}
let r = (|| {
self.declared(fn_name)?;
let asof = self.asof.clone();
let (fn_hash, desc) = self.function(fn_name, &asof, None)?;
let slice = cells_slice(fn_name, &fn_hash, &desc, pattern).map_err(EvalError::InvalidInput)?;
let s = self.slice_read(fn_name, Arc::new(slice), desc, Spec::Cells { pattern: pattern.clone() })?;
Ok(s.members.iter().map(|m| Member::new(m.args.clone(), m.input())).collect())
})();
self.guard(r)
}
fn read_versions(&mut self, fn_name: &str, args: &Args, from: &str, to: &str) -> Result<Vec<Input>, EvalError> {
let times_ok = is_instant(from) && is_instant(to) && to <= self.asof.as_str();
if let Some(e) = self.stopped().filter(|_| times_ok) {
return Err(e);
}
let r = (|| {
instant_arg(from, "$.from")?;
instant_arg(to, "$.to")?;
if to > self.asof.as_str() {
return Err(self.future());
}
self.declared(fn_name)?;
let asof = self.asof.clone();
let (fn_hash, desc) = self.function(fn_name, &asof, None)?;
let (ns, f) = fn_name.rsplit_once('/').unwrap_or(("", fn_name));
let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args: args.clone() };
check_args(&cell, &desc).map_err(EvalError::InvalidInput)?;
if desc.lang != STORED {
return Err(EvalError::InvalidInput(CoreError::new(
ErrorCode::Slice,
"$",
"versions() of a computed function: versions are of stored cells",
)));
}
let slice = versions_slice(&cell, from, to).map_err(EvalError::InvalidInput)?;
let spec = Spec::Versions { to: to.to_string() };
let s = self.slice_read(fn_name, Arc::new(slice), desc, spec)?;
Ok(s.members.iter().map(MemberRec::input).collect())
})();
self.guard(r)
}
fn binding(&mut self, fn_name: &str, at: Option<&str>) -> Result<Option<FnBinding>, EvalError> {
if let Some(e) = self.stopped() {
return Err(e);
}
let r = (|| {
self.declared(fn_name)?;
if let Some(t) = at {
instant_arg(t, "$.at")?;
}
let when = at.unwrap_or(&self.asof).to_string();
let Some((fn_hash, start)) = self.req.binding(fn_name, &when)? else { return Ok(None) };
let (_, descriptor) = self.function(fn_name, &when, at)?;
Ok(Some(FnBinding { fn_hash, start, descriptor }))
})();
self.guard(r)
}
fn asof(&mut self) -> String {
self.asof_read = true;
self.asof.clone()
}
fn binding_peek(&mut self, fn_name: &str, at: Option<&str>) -> Result<Option<FnBinding>, EvalError> {
if let Some(e) = self.stopped() {
return Err(e);
}
let r = (|| {
self.declared(fn_name)?;
if let Some(t) = at {
instant_arg(t, "$.at")?;
}
let when = at.unwrap_or(&self.asof).to_string();
let Some((fn_hash, start)) = self.req.binding(fn_name, &when)? else { return Ok(None) };
let descriptor = match self.req.eng.descriptor_of(&fn_hash) {
Ok(Some(d)) => d,
Ok(None) => {
return Err(EvalError::Store(StoreError::new(
"descriptor",
datumesh_store::Error::NotFound { what: "descriptor", key: fn_hash },
)))
}
Err(EngineError::Store(e)) => return Err(EvalError::Store(e)),
Err(EngineError::Invalid(e)) => return Err(EvalError::InvalidInput(e)),
Err(e) => return Err(EvalError::Runtime(e.to_string())),
};
Ok(Some(FnBinding::new(fn_hash, start, descriptor)))
})();
self.guard(r)
}
fn read_blob(&mut self, sha256: &str, length: u64) -> Result<Vec<u8>, EvalError> {
if let Some(e) = self.stopped() {
return Err(e);
}
let r = (|| {
lexical::check_hex(sha256, 64, "$.sha256").map_err(EvalError::InvalidInput)?;
let unavailable = |reason: String| EvalError::BlobUnavailable { sha256: sha256.to_string(), reason };
let Some(bytes) = self.req.eng.store.blob(sha256).map_err(store_err("blob"))? else {
return Err(unavailable("the store has no content for it".into()));
};
if bytes.len() as u64 != length {
return Err(unavailable(format!("the content has {} bytes, the blob {length}", bytes.len())));
}
if datumesh_core::sha256_hex(&bytes) != sha256 {
return Err(unavailable("the content has another SHA-256".into()));
}
Ok(bytes.into_owned())
})();
self.guard(r)
}
fn asof_compare(&mut self, t: &str) -> Result<std::cmp::Ordering, EvalError> {
instant_arg(t, "$.t")?;
let ord = t.cmp(self.asof.as_str());
if ord == std::cmp::Ordering::Greater {
self.asof_read = true;
self.future_seen = true;
} else if t > self.compared.as_str() {
self.compared = t.to_string();
}
Ok(ord)
}
}
fn cell_named(fn_name: &str, fn_hash: String, args: &Args) -> Cell {
let (ns, f) = fn_name.rsplit_once('/').unwrap_or(("", fn_name));
Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args: args.clone() }
}
pub(crate) fn exhausted(b: &Budget) -> EngineError {
match b.exhausted() {
crate::runtime::Exhausted::Work => EngineError::WorkExceeded { limit: b.work_limit() },
_ => EngineError::BudgetExceeded { fuel: b.limit() },
}
}
pub(crate) fn engine_error(e: EvalError, cell_id: &str) -> EngineError {
match e {
EvalError::NoRuntime { lang } => EngineError::NoRuntime { lang, fn_hash: String::new() },
EvalError::UndeclaredInput { fn_name } => EngineError::UndeclaredInput { fn_name },
EvalError::Cycle { cell_id } => EngineError::Cycle { cell_id },
EvalError::DepthExceeded { limit } => EngineError::DepthExceeded { limit },
EvalError::BudgetExceeded { fuel } => EngineError::BudgetExceeded { fuel },
EvalError::NameUnresolved { fn_name, asof } => EngineError::NameUnresolved { fn_name, asof },
EvalError::Store(e) => e.into(),
EvalError::InvalidInput(e) => EngineError::Invalid(e),
EvalError::BlobUnavailable { sha256, reason } => EngineError::BlobUnavailable { sha256, reason },
other => EngineError::Runtime { cell_id: cell_id.to_string(), message: other.to_string() },
}
}
pub(crate) fn parse_stored(hash: &str, bytes: &[u8]) -> Result<Parsed, StoreError> {
match Receipt::from_json(bytes) {
Ok(r) if r.hash() == hash => Ok(Parsed { bytes: Arc::new(bytes.to_vec()), receipt: r }),
Ok(_) | Err(_) => Err(StoreError::new(
"receipt",
datumesh_store::Error::Invalid(CoreError::new(
ErrorCode::InputInvalid,
"$",
format!("the store's receipt {hash} does not parse to that hash"),
)),
)),
}
}
pub(crate) fn is_instant(s: &str) -> bool {
lexical::check_instant(s, "$").is_ok()
}