use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use std::time::{Duration, Instant};
use datumesh_core::json::{self, Value};
use datumesh_core::{
lexical, parse_object_key, short_id, Arg, Cell, ClassDescriptor, ClassKey, DescribeTests, ErrorCode,
FunctionAnswer, KeyValue, ObjectKey, Scalar, Slice,
};
use crate::call::{Bindings, BoundClass};
use crate::envelope::{self, Cache, Reply};
use crate::mesh::Inner;
pub(crate) const INDEX_TTL: Duration = Duration::from_secs(300);
pub(crate) const INDEX_TICK: Duration = Duration::from_millis(500);
pub(crate) const INDEX_LIMIT: usize = 1_000_000;
#[derive(Clone, Debug)]
struct Known {
key: ObjectKey,
class_hash: String,
}
#[derive(Default)]
pub(crate) struct Index {
classes: BTreeSet<String>,
announced: Vec<String>,
dirty: bool,
built: Option<Built>,
}
pub(crate) struct Built {
classes: BTreeSet<String>,
at: Instant,
objects: BTreeMap<String, Known>,
cells: BTreeMap<String, String>,
}
impl Index {
pub(crate) fn hear_of_class(&mut self, name: &str) {
if self.classes.insert(name.to_string()) {
self.dirty = true;
}
}
pub(crate) fn announce(&mut self, cell_id: &str) {
if self.announced.len() < 100_000 {
self.announced.push(cell_id.to_string());
}
self.dirty = true;
}
pub(crate) fn note_computed(&mut self, cell: &Cell) {
let id = cell.id();
if self.built.as_ref().is_some_and(|b| b.cells.contains_key(&id)) {
return;
}
self.hear_of_class(&cell.ns);
self.announce(&id);
}
}
fn class_of_cell(inner: &Inner, cell_id: &str) -> Option<String> {
let far = "9999-12-31T23:59:59.999999Z";
let (_, bytes) = inner.store.stored_asof_with_receipt(cell_id, far).ok().flatten()?;
let r = datumesh_core::Receipt::from_json(&bytes).ok()?;
Some(r.cell().ns.clone())
}
fn object_of(class: &str, d: &ClassDescriptor, args: &datumesh_core::Args) -> Option<ObjectKey> {
let mut keys = BTreeMap::new();
for (name, decl) in &d.keys {
let Arg::Value(v) = args.get(name)? else { return None };
let value = match decl {
ClassKey::Class(_) => match v {
Scalar::String(enc) => KeyValue::Object(parse_object_key(enc).ok()?),
_ => return None,
},
_ => KeyValue::Scalar(v.clone()),
};
keys.insert(name.clone(), value);
}
ObjectKey::new(class, keys).ok()
}
impl Inner {
pub(crate) fn hear_of_class(&self, name: &str) {
self.known.lock().unwrap_or_else(|p| p.into_inner()).hear_of_class(name);
}
fn recorded_cells(&self, name: &str, fn_hash: &str, d: &datumesh_core::FunctionDescriptor, now: &str) -> Vec<Cell> {
let Some((ns, f)) = name.rsplit_once('/') else { return vec![] };
let args: datumesh_core::Args = d.signature.args.keys().map(|k| (k.clone(), Arg::Slice)).collect();
let cell = Cell { ns: ns.into(), fn_name: f.into(), fn_hash: fn_hash.into(), args };
let Ok(slice) = Slice::from_value(&cell.to_value()) else { return vec![] };
let mut out = match self.store.slice_values(&slice, now) {
Ok(vs) => vs.into_iter().map(|v| v.cell).collect(),
Err(e) => {
tracing::warn!(target: "datumesh_http", error = %e, "describe index: slice_values");
vec![]
}
};
match self.store.computed_cells(&slice, now, INDEX_LIMIT) {
Ok(cells) => {
if cells.len() >= INDEX_LIMIT {
tracing::warn!(target: "datumesh_http", "describe index: {name} reached the limit");
}
out.extend(cells);
}
Err(e) => tracing::warn!(target: "datumesh_http", error = %e, "describe index: computed_cells"),
}
out
}
pub(crate) fn refresh_index(&self) {
let (mut classes, announced) = {
let mut ix = self.known.lock().unwrap_or_else(|p| p.into_inner());
let due = ix.built.as_ref().is_none_or(|b| b.at.elapsed() > INDEX_TTL) || ix.dirty;
if !due {
return;
}
ix.dirty = false;
(ix.classes.clone(), std::mem::take(&mut ix.announced))
};
let now = self.now();
let built = self.build_index(&mut classes, announced, &now);
let mut ix = self.known.lock().unwrap_or_else(|p| p.into_inner());
ix.classes = classes;
ix.built = Some(built);
}
fn build_index(&self, classes: &mut BTreeSet<String>, announced: Vec<String>, now: &str) -> Built {
for id in announced {
if let Some(c) = class_of_cell(self, &id) {
classes.insert(c);
}
}
for target in self.config.aliases.values() {
if let Some((ns, _)) = target.rsplit_once('/') {
classes.insert(ns.to_string());
}
}
let mut objects = BTreeMap::new();
let mut cells = BTreeMap::new();
for name in classes.iter() {
self.index_class(name, now, &mut objects, &mut cells);
}
Built { classes: classes.clone(), at: Instant::now(), objects, cells }
}
fn index_class(
&self,
name: &str,
now: &str,
objects: &mut BTreeMap<String, Known>,
cells: &mut BTreeMap<String, String>,
) {
let Ok(Some(c)) = self.class_binding(name, now) else { return };
let d = &c.descriptor;
for (m, hash) in &d.methods {
let name_of = format!("{name}/{m}");
let Ok(fd) = self.descriptor(hash) else { continue };
for cell in self.recorded_cells(&name_of, hash, &fd, now) {
let Some(object) = object_of(name, d, &cell.args) else { continue };
let Ok(id) = object.object_id(&c.class_hash) else { continue };
cells.insert(cell.id(), id.clone());
objects.insert(id, Known { key: object, class_hash: c.class_hash.clone() });
}
}
}
pub(crate) fn index_named_class(&self, name: &str) {
let now = self.now();
let covered = {
let mut ix = self.known.lock().unwrap_or_else(|p| p.into_inner());
ix.classes.insert(name.to_string());
ix.built.as_ref().is_none_or(|b| b.classes.contains(name))
};
if covered {
return;
}
let mut objects = BTreeMap::new();
let mut cells = BTreeMap::new();
self.index_class(name, &now, &mut objects, &mut cells);
let mut ix = self.known.lock().unwrap_or_else(|p| p.into_inner());
if let Some(b) = ix.built.as_mut() {
b.objects.extend(objects);
b.cells.extend(cells);
b.classes.insert(name.to_string());
}
}
pub(crate) fn index_cell_now(&self, cell: &Cell) {
let class = cell.ns.clone();
let now = self.now();
let Ok(Some(c)) = self.class_binding(&class, &now) else { return };
let Some(object) = object_of(&class, &c.descriptor, &cell.args) else { return };
let Ok(id) = object.object_id(&c.class_hash) else { return };
let mut ix = self.known.lock().unwrap_or_else(|p| p.into_inner());
ix.classes.insert(class);
if let Some(b) = ix.built.as_mut() {
b.cells.insert(cell.id(), id.clone());
b.objects.insert(id, Known { key: object, class_hash: c.class_hash.clone() });
}
}
pub(crate) fn index_stored_now(&self, cell_id: &str) {
let far = "9999-12-31T23:59:59.999999Z";
let Some((_, bytes)) = self.store.stored_asof_with_receipt(cell_id, far).ok().flatten() else { return };
let Ok(r) = datumesh_core::Receipt::from_json(&bytes) else { return };
self.index_cell_now(r.cell());
}
fn with_index<T>(&self, f: impl FnOnce(&Built) -> T) -> Option<T> {
let ix = self.known.lock().unwrap_or_else(|p| p.into_inner());
ix.built.as_ref().map(f)
}
pub(crate) fn describe(&self, id: &str) -> Reply {
let now = self.now();
if lexical::check_function_name(id, "$").is_ok() {
return match Bindings::class(self, id, &now) {
Ok(Some(c)) => {
self.index_named_class(id);
self.class_answer(id, &c)
}
Ok(None) => {
let method = id
.rsplit_once('/')
.is_some_and(|(ns, _)| matches!(Bindings::class(self, ns, &now), Ok(Some(_))));
if method {
return not_found();
}
match Bindings::binding(self, id, &now) {
Ok(Some(f)) => self.function_answer(id, &f.fn_hash, &f.descriptor),
Ok(None) => not_found(),
Err(code) => envelope::error_reply(code),
}
}
Err(code) => envelope::error_reply(code),
};
}
if id.contains('(') {
return match parse_object_key(id) {
Ok(key) => self.object_by_key(key, &now),
Err(e) => envelope::bad_request_reply(e.code()),
};
}
let hex = |s: &str, n: usize| s.len() == n && lexical::check_hex(s, n, "$").is_ok();
if hex(id, 64) {
let answer = self.with_index(|b| {
let object = b.objects.get(id).or_else(|| b.cells.get(id).and_then(|o| b.objects.get(o)));
match object {
Some(k) => self.object_answer(&k.key, &k.class_hash),
None => not_found(),
}
});
return answer.unwrap_or_else(index_not_yet);
}
if hex(id, 12) {
let answer = self.with_index(|b| {
let mut found: BTreeSet<&String> = BTreeSet::new();
for o in b.objects.keys().filter(|o| o.starts_with(id)) {
found.insert(o);
}
for (c, o) in &b.cells {
if c.starts_with(id) {
found.insert(o);
}
}
let mut it = found.into_iter();
match (it.next(), it.next()) {
(None, _) => not_found(),
(Some(o), None) => match b.objects.get(o) {
Some(k) => self.object_answer(&k.key, &k.class_hash),
None => not_found(),
},
(Some(_), Some(_)) => envelope::bad_request_reply(ErrorCode::ShortIdAmbiguous),
}
});
return answer.unwrap_or_else(index_not_yet);
}
not_found()
}
fn function_answer(&self, name: &str, fn_hash: &str, d: &datumesh_core::FunctionDescriptor) -> Reply {
let Ok(answer) = FunctionAnswer::new(name, d) else {
return envelope::error_reply(ErrorCode::InputUnavailable);
};
let unnamed = self.is_unnamed(fn_hash);
let counts = crate::testrun::counts(&*self.store, name, fn_hash, unnamed, &self.now());
let Ok((count, branches, covered, counterexamples)) = counts else {
return envelope::error_reply(ErrorCode::InputUnavailable);
};
let tests = DescribeTests { count, branches, covered, counterexamples };
let Ok(answer) = answer.with_tests(tests) else {
return envelope::error_reply(ErrorCode::InputUnavailable);
};
let Ok(mut answer) = answer.with_publisher(publisher_of(name)) else {
return envelope::error_reply(ErrorCode::InputUnavailable);
};
if let Some(relay) = crate::fnpage::relay_of(self, name, fn_hash) {
if let Some(takeover) = relay.takeover {
let source =
datumesh_core::DescribeSource { name: relay.name, relayed_by: self.config.host.clone(), takeover };
if let Ok(a) = answer.clone().with_source(source) {
answer = a;
}
}
}
Reply::json(200, answer.to_json(), Cache::NoStore)
}
fn class_answer(&self, name: &str, c: &BoundClass) -> Reply {
let Ok(descriptor) = json::parse(&c.descriptor.canonical()) else {
return envelope::error_reply(ErrorCode::InputUnavailable);
};
let mut methods = Vec::new();
for (m, hash) in &c.descriptor.methods {
let Ok(fd) = self.descriptor(hash) else { return envelope::error_reply(ErrorCode::InputUnavailable) };
let fv = fd.to_value();
let sig = fv.get("signature");
let args: Vec<(String, Value)> = match sig.and_then(|s| s.get("args")) {
Some(Value::Object(a)) => {
a.iter().filter(|(k, _)| !c.descriptor.keys.contains_key(k)).cloned().collect()
}
_ => vec![],
};
let returns = sig.and_then(|s| s.get("returns")).cloned().unwrap_or(Value::Null);
let mut members = vec![("args".into(), Value::Object(args))];
if let Some(text) =
self.config.catalog.get(&format!("{name}/{m}")).filter(|t| (1..=1024).contains(&t.len()))
{
members.push(("description".into(), Value::String(text.clone())));
}
members.push(("fn".into(), Value::String(format!("{name}/{m}"))));
members.push(("fn_hash".into(), Value::String(hash.clone())));
members.push(("returns".into(), returns));
methods.push((m.clone(), Value::Object(members)));
}
let doc = Value::Object(vec![
("class".into(), Value::String(name.into())),
("class_hash".into(), Value::String(c.class_hash.clone())),
("descriptor".into(), descriptor),
("kind".into(), Value::String("class".into())),
("methods".into(), Value::Object(methods)),
("publisher".into(), Value::String(publisher_of(name).into())),
]);
Reply::json(200, doc.to_jcs(), Cache::Public(self.config.ttl.latest_max_age.as_secs()))
}
fn object_by_key(&self, key: ObjectKey, now: &str) -> Reply {
let c = match Bindings::class(self, &key.class, now) {
Ok(Some(c)) => c,
Ok(None) => return not_found(),
Err(code) => return envelope::error_reply(code),
};
let d = &c.descriptor;
let bad = |code| envelope::bad_request_reply(code);
if key.keys.len() != d.keys.len() {
return bad(ErrorCode::KeyEncoding);
}
for (name, decl) in &d.keys {
let Some(v) = key.keys.get(name) else { return bad(ErrorCode::KeyEncoding) };
match (decl, v) {
(ClassKey::Class(want), KeyValue::Object(o)) if o.class == *want => {}
(ClassKey::Scalar { ty, key_normalisation, .. }, KeyValue::Scalar(s)) if s.ty() == *ty => {
if let Some(rule) = key_normalisation {
if let Err(e) = rule.check(s.lexical()) {
return bad(e.code());
}
}
}
_ => return bad(ErrorCode::KeyEncoding),
}
}
self.object_answer(&key, &c.class_hash)
}
fn object_answer(&self, key: &ObjectKey, class_hash: &str) -> Reply {
let class: Option<Arc<ClassDescriptor>> =
self.store.class(class_hash).ok().flatten().and_then(|b| ClassDescriptor::from_json(&b).ok()).map(Arc::new);
let Some(class) = class else { return envelope::error_reply(ErrorCode::InputUnavailable) };
let (Ok(object), Ok(id)) = (key.object_value(class_hash), key.object_id(class_hash)) else {
return bad_key();
};
let Ok(short) = short_id(&id) else { return bad_key() };
let doc = Value::Object(vec![
("call".into(), Value::String(key.call_form())),
("kind".into(), Value::String("object".into())),
("methods".into(), Value::Array(class.methods.keys().map(|m| Value::String(m.clone())).collect())),
("object".into(), object),
("object_id".into(), Value::String(id)),
("short_id".into(), Value::String(short)),
]);
Reply::json(200, doc.to_jcs(), Cache::Public(self.config.ttl.latest_max_age.as_secs()))
}
}
pub(crate) fn publisher_of(name: &str) -> &str {
name.split('/').next().unwrap_or(name)
}
fn bad_key() -> Reply {
envelope::bad_request_reply(ErrorCode::KeyEncoding)
}
fn index_not_yet() -> Reply {
envelope::not_yet_reply(INDEX_TTL.as_secs(), "The index of objects")
}
fn not_found() -> Reply {
let body = Value::Object(vec![("error".into(), Value::String("no such class or object".into()))]);
Reply::json(404, body.to_jcs(), Cache::Public(60))
}