use std::collections::BTreeMap;
use std::path::Path;
use anyhow::{anyhow, bail, Context, Result};
use sha2::{Digest, Sha256};
use serde::Deserialize;
use wasmer::{Function, Imports, Instance, Memory, Module, Store, TypedFunction, Value};
#[derive(Debug, Clone, Deserialize)]
pub struct Provenance {
pub language: String,
pub grammar_name: String,
pub vocabulary: String,
pub generate_cli: String,
pub runtime: String,
pub pack_abi: u32,
#[serde(default)]
pub versions: String,
#[serde(default)]
pub sources: BTreeMap<String, String>,
#[serde(default)]
pub sweeps: serde_json::Value,
}
#[derive(Debug, Clone, Deserialize)]
pub struct PackRoles {
#[serde(default)]
pub facets: BTreeMap<String, Vec<String>>,
}
const NAMED: u32 = 1;
const IS_ERROR: u32 = 2;
const HAS_ERROR: u32 = 4;
const MISSING: u32 = 8;
pub struct Pack {
store: std::cell::RefCell<Store>,
memory: Memory,
f: Abi,
provenance: Provenance,
roles: PackRoles,
roles_raw: String,
node_types_raw: Option<String>,
node_types: std::cell::OnceCell<Option<crate::node_types::NodeTypes>>,
}
struct Abi {
strlen: TypedFunction<u32, u32>,
alloc: TypedFunction<u32, u32>,
free: TypedFunction<u32, ()>,
parse: TypedFunction<(u32, u32), u32>,
tree_free: TypedFunction<u32, ()>,
tree_root: TypedFunction<(u32, u32), ()>,
node_new: TypedFunction<(), u32>,
node_free: TypedFunction<u32, ()>,
node_type: TypedFunction<u32, u32>,
node_sexp: TypedFunction<u32, u32>,
node_flags: TypedFunction<u32, u32>,
node_start_byte: TypedFunction<u32, u32>,
node_end_byte: TypedFunction<u32, u32>,
node_child_count: TypedFunction<u32, u32>,
node_child: TypedFunction<(u32, u32, u32), u32>,
node_named_child_count: TypedFunction<u32, u32>,
node_named_child: TypedFunction<(u32, u32, u32), u32>,
field_name_for_child: TypedFunction<(u32, u32), u32>,
cstr_free: TypedFunction<u32, ()>,
query: Option<QueryAbi>,
}
struct QueryAbi {
new: TypedFunction<(u32, u32, u32, u32), u32>,
delete: TypedFunction<u32, ()>,
exec: TypedFunction<(u32, u32), u32>,
cursor_delete: TypedFunction<u32, ()>,
next_capture: TypedFunction<(u32, u32, u32, u32, u32), u32>,
capture_name: TypedFunction<(u32, u32), u32>,
}
impl Pack {
pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref();
let bytes = std::fs::read(path)
.with_context(|| format!("reading {}", path.display()))?;
Self::from_bytes(&bytes)
}
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
let mut store = Store::default();
let module = compile_cached(&store, bytes)?;
let imports = refuse_every_import(&module, &mut store)?;
let instance =
Instance::new(&mut store, &module, &imports).context("instantiating the pack")?;
instance
.exports
.get_typed_function::<(), ()>(&store, "_initialize")
.context("pack has no _initialize; is it a treebank pack?")?
.call(&mut store)?;
let memory = instance
.exports
.get_memory("memory")
.map_err(|_| anyhow!("pack exports no memory"))?
.clone();
let f = Abi::bind(&instance, &mut store)?;
let provenance = read_json(&mut store, &memory, &instance, "tb_provenance")?;
let roles_raw = read_string(&mut store, &memory, &instance, "tb_roles")?;
let roles = serde_json::from_str(&roles_raw).context("parsing tb_roles")?;
let node_types_raw = read_string(&mut store, &memory, &instance, "tb_node_types").ok();
Ok(Self {
store: std::cell::RefCell::new(store),
memory,
f,
provenance,
roles,
roles_raw,
node_types_raw,
node_types: std::cell::OnceCell::new(),
})
}
pub fn provenance(&self) -> &Provenance {
&self.provenance
}
pub fn roles(&self) -> &PackRoles {
&self.roles
}
pub fn roles_json(&self) -> &str {
&self.roles_raw
}
pub fn node_types_json(&self) -> Option<&str> {
self.node_types_raw.as_deref()
}
pub fn node_types(&self) -> Option<&crate::node_types::NodeTypes> {
self.node_types
.get_or_init(|| {
let raw = self.node_types_raw.as_deref()?;
crate::node_types::NodeTypes::parse(raw).ok()
})
.as_ref()
}
pub fn language(&self) -> &str {
&self.provenance.language
}
pub fn expand_query(&self, query: &str) -> Result<String> {
crate::expand::expand_with_types(query, &self.roles.facets, self.node_types())
}
pub fn parse(&self, source: &str) -> Result<Tree<'_>> {
let bytes = source.as_bytes();
let len = u32::try_from(bytes.len()).context("source larger than 4 GiB")?;
let mut store = self.store.borrow_mut();
let ptr = self.f.alloc.call(&mut *store, len)?;
if ptr == 0 {
return Err(anyhow!("the pack could not allocate {len} bytes"));
}
self.memory
.view(&*store)
.write(ptr as u64, bytes)
.context("writing source into the pack")?;
let tree = self.f.parse.call(&mut *store, ptr, len)?;
self.f.free.call(&mut *store, ptr)?;
if tree == 0 {
return Err(anyhow!("parse failed"));
}
Ok(Tree { pack: self, handle: tree })
}
}
fn refuse_every_import(module: &Module, store: &mut Store) -> Result<Imports> {
let mut imports = Imports::new();
for import in module.imports() {
if import.module() != "wasi_snapshot_preview1" {
bail!(
"a pack may import only WASI, but this one imports {}::{}",
import.module(),
import.name()
);
}
let signature = match import.ty() {
wasmer::ExternType::Function(signature) => signature.clone(),
other => bail!("a pack imports a non-function {other:?}"),
};
imports.define(
import.module(),
import.name(),
Function::new(store, &signature, |_: &[Value]| Ok(vec![Value::I32(WASI_BADF)])),
);
}
Ok(imports)
}
const WASI_BADF: i32 = 8;
pub struct Tree<'p> {
pack: &'p Pack,
handle: u32,
}
impl<'p> Tree<'p> {
pub fn root(&self) -> Node<'p> {
let mut store = self.pack.store.borrow_mut();
let node = self.pack.f.node_new.call(&mut *store).expect("tb_node_new");
self.pack
.f
.tree_root
.call(&mut *store, self.handle, node)
.expect("tb_tree_root");
Node { pack: self.pack, handle: node }
}
}
impl Drop for Tree<'_> {
fn drop(&mut self) {
let mut store = self.pack.store.borrow_mut();
let _ = self.pack.f.tree_free.call(&mut *store, self.handle);
}
}
pub struct Node<'p> {
pack: &'p Pack,
handle: u32,
}
impl<'p> Node<'p> {
pub fn kind(&self) -> Result<String> {
let mut store = self.pack.store.borrow_mut();
let ptr = self.pack.f.node_type.call(&mut *store, self.handle)?;
read_cstr(&mut store, &self.pack.memory, &self.pack.f, ptr)
}
pub fn sexp(&self) -> Result<String> {
let mut store = self.pack.store.borrow_mut();
let ptr = self.pack.f.node_sexp.call(&mut *store, self.handle)?;
let out = read_cstr(&mut store, &self.pack.memory, &self.pack.f, ptr)?;
self.pack.f.cstr_free.call(&mut *store, ptr)?;
Ok(out)
}
pub fn byte_range(&self) -> Result<std::ops::Range<usize>> {
let mut store = self.pack.store.borrow_mut();
let start = self.pack.f.node_start_byte.call(&mut *store, self.handle)?;
let end = self.pack.f.node_end_byte.call(&mut *store, self.handle)?;
Ok(start as usize..end as usize)
}
pub fn is_named(&self) -> Result<bool> {
Ok(self.flags()? & NAMED != 0)
}
pub fn is_error(&self) -> Result<bool> {
Ok(self.flags()? & (IS_ERROR | MISSING) != 0)
}
pub fn has_error(&self) -> Result<bool> {
Ok(self.flags()? & (HAS_ERROR | IS_ERROR | MISSING) != 0)
}
fn flags(&self) -> Result<u32> {
let mut store = self.pack.store.borrow_mut();
Ok(self.pack.f.node_flags.call(&mut *store, self.handle)?)
}
pub fn child_count(&self, named_only: bool) -> Result<u32> {
let mut store = self.pack.store.borrow_mut();
let f = if named_only {
&self.pack.f.node_named_child_count
} else {
&self.pack.f.node_child_count
};
Ok(f.call(&mut *store, self.handle)?)
}
pub fn child(&self, index: u32, named_only: bool) -> Result<Option<Node<'p>>> {
let mut store = self.pack.store.borrow_mut();
let kid = self.pack.f.node_new.call(&mut *store)?;
let f = if named_only { &self.pack.f.node_named_child } else { &self.pack.f.node_child };
let ok = f.call(&mut *store, self.handle, index, kid)?;
if ok == 0 {
self.pack.f.node_free.call(&mut *store, kid)?;
return Ok(None);
}
Ok(Some(Node { pack: self.pack, handle: kid }))
}
pub fn field_name_for_child(&self, index: u32) -> Result<Option<String>> {
let mut store = self.pack.store.borrow_mut();
let ptr = self.pack.f.field_name_for_child.call(&mut *store, self.handle, index)?;
if ptr == 0 {
return Ok(None);
}
Ok(Some(read_cstr(&mut store, &self.pack.memory, &self.pack.f, ptr)?))
}
pub fn named_children(&self) -> Result<Vec<Node<'p>>> {
let count = self.child_count(true)?;
(0..count).filter_map(|i| self.child(i, true).transpose()).collect()
}
}
impl Drop for Node<'_> {
fn drop(&mut self) {
let mut store = self.pack.store.borrow_mut();
let _ = self.pack.f.node_free.call(&mut *store, self.handle);
}
}
#[derive(Debug, Clone)]
pub struct Capture {
pub name: String,
pub kind: String,
pub range: std::ops::Range<usize>,
pub pattern: u32,
}
impl Pack {
pub fn query(&self, tree: &Tree<'_>, query: &str) -> Result<Vec<Capture>> {
let expanded = self.expand_query(query)?;
let root = tree.root();
self.query_node(&root, &expanded)
}
pub fn query_node(&self, node: &Node<'_>, expanded: &str) -> Result<Vec<Capture>> {
let q = self.f.query.as_ref().ok_or_else(|| {
anyhow!(
"this {} pack cannot run queries: it is pack_abi {}, and queries need 3. \
Fetch a current pack, or use expand_query and your own query engine.",
self.provenance.language, self.provenance.pack_abi
)
})?;
let bytes = expanded.as_bytes();
let len = u32::try_from(bytes.len()).context("query larger than 4 GiB")?;
let mut store = self.store.borrow_mut();
let src = self.f.alloc.call(&mut *store, len)?;
let errs = self.f.alloc.call(&mut *store, 8)?;
if src == 0 || errs == 0 {
return Err(anyhow!("the pack could not allocate for a query"));
}
self.memory.view(&*store).write(src as u64, bytes)?;
let handle = q.new.call(&mut *store, src, len, errs, errs + 4)?;
if handle == 0 {
let mut raw = [0u8; 8];
self.memory.view(&*store).read(errs as u64, &mut raw)?;
let offset = u32::from_le_bytes(raw[0..4].try_into().unwrap()) as usize;
let kind = u32::from_le_bytes(raw[4..8].try_into().unwrap());
self.f.free.call(&mut *store, src)?;
self.f.free.call(&mut *store, errs)?;
return Err(anyhow!(
"{} at byte {offset} of the query:\n {expanded}\n {:>offset$}^",
query_error(kind),
""
));
}
self.f.free.call(&mut *store, src)?;
self.f.free.call(&mut *store, errs)?;
let cursor = q.exec.call(&mut *store, handle, node.handle)?;
if cursor == 0 {
q.delete.call(&mut *store, handle)?;
return Err(anyhow!("the pack could not allocate a query cursor"));
}
let out_node = self.f.node_new.call(&mut *store)?;
let out = self.f.alloc.call(&mut *store, 8)?;
let mut found = Vec::new();
let mut names: std::collections::HashMap<u32, String> = std::collections::HashMap::new();
loop {
let more = q
.next_capture
.call(&mut *store, cursor, handle, out_node, out, out + 4)?;
if more == 0 {
break;
}
let mut raw = [0u8; 8];
self.memory.view(&*store).read(out as u64, &mut raw)?;
let pattern = u32::from_le_bytes(raw[0..4].try_into().unwrap());
let capture = u32::from_le_bytes(raw[4..8].try_into().unwrap());
let kind_ptr = self.f.node_type.call(&mut *store, out_node)?;
let kind = read_cstr(&mut store, &self.memory, &self.f, kind_ptr)?;
let start = self.f.node_start_byte.call(&mut *store, out_node)? as usize;
let end = self.f.node_end_byte.call(&mut *store, out_node)? as usize;
let name = match names.get(&capture) {
Some(name) => name.clone(),
None => {
let ptr = q.capture_name.call(&mut *store, handle, capture)?;
let name = read_cstr(&mut store, &self.memory, &self.f, ptr)?;
names.insert(capture, name.clone());
name
}
};
found.push(Capture { name, kind, range: start..end, pattern });
}
self.f.free.call(&mut *store, out)?;
self.f.node_free.call(&mut *store, out_node)?;
q.cursor_delete.call(&mut *store, cursor)?;
q.delete.call(&mut *store, handle)?;
Ok(found)
}
}
fn query_error(kind: u32) -> &'static str {
match kind {
1 => "the query is not valid s-expression syntax",
2 => "the query names a node type this grammar does not have",
3 => "the query names a field this grammar does not have",
4 => "the query captures something that cannot be captured",
5 => "the query asks for a shape this grammar cannot produce, \
usually a field on a node type that does not have it",
6 => "the query names a language that is not this one",
_ => "the query is not valid",
}
}
#[cfg(test)]
mod query_error_tests {
use super::query_error;
#[test]
fn the_numbering_matches_the_header() {
assert!(query_error(1).contains("s-expression"));
assert!(query_error(2).contains("node type"));
assert!(query_error(3).contains("field"));
assert!(query_error(4).contains("captures"));
assert!(query_error(5).contains("shape"));
assert!(query_error(6).contains("language"));
assert_eq!(query_error(7), "the query is not valid");
assert_eq!(query_error(99), "the query is not valid");
}
}
fn read_cstr(
store: &mut Store,
memory: &Memory,
f: &Abi,
ptr: u32,
) -> Result<String> {
if ptr == 0 {
return Ok(String::new());
}
let len = f.strlen.call(&mut *store, ptr)? as usize;
let mut buf = vec![0u8; len];
memory.view(&*store).read(ptr as u64, &mut buf)?;
Ok(String::from_utf8(buf)?)
}
fn read_json<T: for<'de> Deserialize<'de>>(
store: &mut Store,
memory: &Memory,
instance: &Instance,
name: &str,
) -> Result<T> {
let ptr = instance
.exports
.get_typed_function::<(), u32>(&*store, name)
.with_context(|| format!("pack has no {name}"))?
.call(&mut *store)?;
let len = instance
.exports
.get_typed_function::<(), u32>(&*store, &format!("{name}_len"))
.with_context(|| format!("pack has no {name}_len"))?
.call(&mut *store)? as usize;
let mut buf = vec![0u8; len];
memory.view(&*store).read(ptr as u64, &mut buf)?;
serde_json::from_slice(&buf).with_context(|| format!("parsing {name}"))
}
fn read_string(
store: &mut Store,
memory: &Memory,
instance: &Instance,
name: &str,
) -> Result<String> {
let ptr = instance
.exports
.get_typed_function::<(), u32>(&*store, name)
.with_context(|| format!("pack has no {name}"))?
.call(&mut *store)?;
let len = instance
.exports
.get_typed_function::<(), u32>(&*store, &format!("{name}_len"))
.with_context(|| format!("pack has no {name}_len"))?
.call(&mut *store)? as usize;
let mut buf = vec![0u8; len];
memory.view(&*store).read(ptr as u64, &mut buf)?;
String::from_utf8(buf).with_context(|| format!("{name} is not utf-8"))
}
impl QueryAbi {
fn bind(instance: &Instance, store: &mut Store) -> Option<Self> {
Some(Self {
new: instance.exports.get_typed_function(&*store, "tb_query_new").ok()?,
delete: instance.exports.get_typed_function(&*store, "tb_query_delete").ok()?,
exec: instance.exports.get_typed_function(&*store, "tb_query_exec").ok()?,
cursor_delete: instance.exports.get_typed_function(&*store, "tb_query_cursor_delete").ok()?,
next_capture: instance.exports.get_typed_function(&*store, "tb_query_next_capture").ok()?,
capture_name: instance.exports.get_typed_function(&*store, "tb_query_capture_name").ok()?,
})
}
}
impl Abi {
fn bind(instance: &Instance, store: &mut Store) -> Result<Self> {
macro_rules! f {
($name:literal) => {
instance
.exports
.get_typed_function(&*store, $name)
.with_context(|| format!("pack has no {}", $name))?
};
}
Ok(Self {
strlen: f!("tb_strlen"),
alloc: f!("tb_alloc"),
free: f!("tb_free"),
parse: f!("tb_parse"),
tree_free: f!("tb_tree_free"),
tree_root: f!("tb_tree_root"),
node_new: f!("tb_node_new"),
node_free: f!("tb_node_free"),
node_type: f!("tb_node_type"),
node_sexp: f!("tb_node_sexp"),
node_flags: f!("tb_node_flags"),
node_start_byte: f!("tb_node_start_byte"),
node_end_byte: f!("tb_node_end_byte"),
node_child_count: f!("tb_node_child_count"),
node_child: f!("tb_node_child"),
node_named_child_count: f!("tb_node_named_child_count"),
node_named_child: f!("tb_node_named_child"),
field_name_for_child: f!("tb_node_field_name_for_child"),
cstr_free: f!("tb_cstr_free"),
query: QueryAbi::bind(instance, store),
})
}
}
fn compile_cached(store: &Store, bytes: &[u8]) -> Result<Module> {
if std::env::var_os("TREEBANK_NO_COMPILE_CACHE").is_some() {
return Module::new(store, bytes).context("not a valid wasm module");
}
let key = {
let mut hasher = Sha256::new();
hasher.update(bytes);
hasher.update(std::env::consts::ARCH.as_bytes());
hasher.update(std::env::consts::OS.as_bytes());
hasher.finalize().iter().map(|b| format!("{b:02x}")).collect::<String>()
};
let path = compile_cache_dir().join(format!("{}.cwasm", &key[..32]));
if path.is_file() {
if let Ok(module) = unsafe { Module::deserialize_from_file(store, &path) } {
return Ok(module);
}
let _ = std::fs::remove_file(&path);
}
let module = Module::new(store, bytes).context("not a valid wasm module")?;
if let Ok(artifact) = module.serialize() {
if let Some(dir) = path.parent() {
if std::fs::create_dir_all(dir).is_ok() {
let tmp = dir.join(format!(".{}.cwasm", std::process::id()));
if std::fs::write(&tmp, &artifact).is_ok() {
let _ = std::fs::rename(&tmp, &path);
}
}
}
}
Ok(module)
}
fn compile_cache_dir() -> std::path::PathBuf {
if let Ok(dir) = std::env::var("TREEBANK_CACHE") {
return std::path::PathBuf::from(dir).join("compiled");
}
if let Ok(dir) = std::env::var("XDG_CACHE_HOME") {
return std::path::PathBuf::from(dir).join("treebank").join("compiled");
}
if let Ok(home) = std::env::var("HOME") {
return std::path::PathBuf::from(home).join(".cache").join("treebank").join("compiled");
}
std::env::temp_dir().join("treebank").join("compiled")
}