use crate::peg::{self, Grammar, Peg};
use crate::{BuildError, Diagnostic, ParseTree, RawValue, serde_bridge};
use dsl_kit_core::IdGen;
use dsl_kit_schema::NodeSchema;
use serde_json::Value;
use std::collections::BTreeMap;
use std::fmt;
pub const IMPORT_VARIANT: &str = "$import";
pub const IMPORT_SPEC_FIELD: &str = "spec";
pub mod import_codes {
pub const CYCLE: &str = "dsl_kit::parse::import::cycle";
pub const DEPTH_EXCEEDED: &str = "dsl_kit::parse::import::depth_exceeded";
pub const SOURCE_LIMIT: &str = "dsl_kit::parse::import::source_limit";
pub const BYTE_LIMIT: &str = "dsl_kit::parse::import::byte_limit";
pub const RESOLVE_FAILED: &str = "dsl_kit::parse::import::resolve_failed";
pub const FETCH_FAILED: &str = "dsl_kit::parse::import::fetch_failed";
pub const SPEC_SHAPE: &str = "dsl_kit::parse::import::spec_shape";
pub const IN_IMPORT: &str = "dsl_kit::parse::import::in_import";
pub const UNEXPANDED: &str = "dsl_kit::parse::import::unexpanded";
pub const TEXT_UNSUPPORTED: &str = "dsl_kit::parse::import::text_unsupported";
pub const BAD_SOURCES: &str = "dsl_kit::parse::import::bad_sources";
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct SourceId(String);
impl SourceId {
pub fn new(id: impl Into<String>) -> Self {
Self(id.into())
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Display for SourceId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.0)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum ImportSource {
Json(String),
Text(String),
}
pub trait SourceResolver {
fn resolve(&mut self, importer: Option<&SourceId>, spec: &str) -> Result<SourceId, String>;
fn fetch(&mut self, id: &SourceId) -> Result<ImportSource, String>;
}
#[derive(Debug, Clone, Default)]
pub struct MapResolver {
sources: BTreeMap<String, ImportSource>,
}
impl MapResolver {
pub fn new() -> Self {
Self::default()
}
pub fn insert(&mut self, name: impl Into<String>, json: impl Into<String>) {
self.sources
.insert(name.into(), ImportSource::Json(json.into()));
}
pub fn insert_text(&mut self, name: impl Into<String>, text: impl Into<String>) {
self.sources
.insert(name.into(), ImportSource::Text(text.into()));
}
pub fn from_sources_json(sources_json: &str) -> Result<Self, BuildError> {
let value: Value = serde_json::from_str(sources_json).map_err(|e| {
BuildError::single(Diagnostic::error(
import_codes::BAD_SOURCES,
format!("sources bundle is not valid JSON: {e}"),
))
})?;
let Value::Object(map) = value else {
return Err(BuildError::single(Diagnostic::error(
import_codes::BAD_SOURCES,
"sources bundle must be a JSON object mapping names to \
single-key {\"json\": …} / {\"text\": …} objects",
)));
};
let mut resolver = Self::new();
let mut diags = Vec::new();
for (name, entry) in map {
let bad = |what: &str| {
Diagnostic::error(
import_codes::BAD_SOURCES,
format!(
"source `{name}` {what} (expected a single-key \
{{\"json\": \"…\"}} or {{\"text\": \"…\"}} object)"
),
)
};
let Value::Object(entry) = entry else {
diags.push(bad("is not an object"));
continue;
};
if entry.len() != 1 {
diags.push(bad("must carry exactly one front-end key"));
continue;
}
let (kind, payload) = entry.iter().next().expect("len checked");
let Value::String(payload) = payload else {
diags.push(bad("has a non-string payload"));
continue;
};
match kind.as_str() {
"json" => resolver.insert(name, payload.clone()),
"text" => resolver.insert_text(name, payload.clone()),
other => diags.push(bad(&format!("uses unknown front-end tag `{other}`"))),
}
}
if diags.is_empty() {
Ok(resolver)
} else {
Err(BuildError::new(diags))
}
}
}
impl SourceResolver for MapResolver {
fn resolve(&mut self, _importer: Option<&SourceId>, spec: &str) -> Result<SourceId, String> {
Ok(SourceId::new(spec))
}
fn fetch(&mut self, id: &SourceId) -> Result<ImportSource, String> {
match self.sources.get(id.as_str()) {
Some(source) => Ok(source.clone()),
None => Err(format!("no source registered under `{id}`")),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ImportLimits {
pub max_depth: usize,
pub max_sources: usize,
pub max_total_bytes: usize,
}
impl Default for ImportLimits {
fn default() -> Self {
Self {
max_depth: 32,
max_sources: 256,
max_total_bytes: 8 * 1024 * 1024,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Loaded {
pub tree: ParseTree,
pub dependencies: Vec<SourceId>,
}
impl Loaded {
pub fn digest(&self) -> String {
let mut h = Fnv1a::new();
feed_tree(&mut h, &self.tree);
for d in &self.dependencies {
h.write(b"\x1fdep\x1f");
h.write(d.as_str().as_bytes());
}
format!("{:016x}", h.finish())
}
}
struct Fnv1a(u64);
impl Fnv1a {
fn new() -> Self {
Self(0xcbf2_9ce4_8422_2325)
}
fn write(&mut self, bytes: &[u8]) {
for &b in bytes {
self.0 ^= u64::from(b);
self.0 = self.0.wrapping_mul(0x0000_0100_0000_01b3);
}
}
fn finish(&self) -> u64 {
self.0
}
}
fn feed_tree(h: &mut Fnv1a, tree: &ParseTree) {
h.write(b"\x1fnode\x1f");
h.write(tree.variant.as_bytes());
for (name, value) in &tree.fields {
h.write(b"\x1ffield\x1f");
h.write(name.as_bytes());
h.write(b"\x1f");
match value {
RawValue::Text(t) => {
h.write(b"t\x1f");
h.write(t.as_bytes());
}
RawValue::Json(v) => {
h.write(b"j\x1f");
h.write(v.to_string().as_bytes());
}
}
}
for (slot, children) in &tree.children {
h.write(b"\x1fslot\x1f");
h.write(slot.as_bytes());
for c in children {
feed_tree(h, c);
}
}
for (slot, entries) in &tree.keyed_children {
h.write(b"\x1fkeyed\x1f");
h.write(slot.as_bytes());
for (key, c) in entries {
h.write(b"\x1fkey\x1f");
h.write(key.as_bytes());
feed_tree(h, c);
}
}
h.write(b"\x1fend\x1f");
}
#[derive(Debug, Clone)]
pub struct Loader<'a> {
schema: &'a NodeSchema,
grammar: Option<&'a Grammar>,
limits: ImportLimits,
}
impl<'a> Loader<'a> {
pub fn new(schema: &'a NodeSchema) -> Self {
Self {
schema,
grammar: None,
limits: ImportLimits::default(),
}
}
pub fn with_grammar(mut self, grammar: &'a Grammar) -> Self {
self.grammar = Some(grammar);
self
}
pub fn with_limits(mut self, limits: ImportLimits) -> Self {
self.limits = limits;
self
}
pub fn load_json_str(
&self,
root: &str,
resolver: &mut dyn SourceResolver,
) -> Result<Loaded, BuildError> {
let tree = serde_bridge::from_json_str(root, self.schema)?;
self.expand_root(tree, resolver)
}
pub fn load_json_value(
&self,
root: &Value,
resolver: &mut dyn SourceResolver,
) -> Result<Loaded, BuildError> {
let tree = serde_bridge::from_json_value(root, self.schema)?;
self.expand_root(tree, resolver)
}
pub fn load_text(
&self,
root: &str,
resolver: &mut dyn SourceResolver,
) -> Result<Loaded, BuildError> {
let Some(grammar) = self.grammar else {
return Err(BuildError::single(Diagnostic::error(
import_codes::TEXT_UNSUPPORTED,
"cannot load a text root: this Loader has no grammar — \
configure one with `Loader::with_grammar`",
)));
};
let tree = grammar.parse(root)?;
self.expand_root(tree, resolver)
}
fn expand_root(
&self,
tree: ParseTree,
resolver: &mut dyn SourceResolver,
) -> Result<Loaded, BuildError> {
let mut state = LoadState {
resolver,
limits: &self.limits,
schema: self.schema,
grammar: self.grammar,
cache: BTreeMap::new(),
stack: Vec::new(),
total_bytes: 0,
};
let tree = expand(tree, None, 0, &mut state).map_err(BuildError::new)?;
let dependencies = state.cache.into_keys().collect();
Ok(Loaded { tree, dependencies })
}
}
pub fn load_json_str(
root: &str,
schema: &NodeSchema,
resolver: &mut dyn SourceResolver,
limits: &ImportLimits,
) -> Result<Loaded, BuildError> {
Loader::new(schema)
.with_limits(limits.clone())
.load_json_str(root, resolver)
}
pub fn load_json_value(
root: &Value,
schema: &NodeSchema,
resolver: &mut dyn SourceResolver,
limits: &ImportLimits,
) -> Result<Loaded, BuildError> {
Loader::new(schema)
.with_limits(limits.clone())
.load_json_value(root, resolver)
}
pub fn add_import_syntax(grammar: &mut Grammar, ids: &IdGen) -> Result<(), BuildError> {
let already = grammar
.rules
.iter()
.any(|r| matches!(r, Peg::Rule { name, .. } if name == IMPORT_VARIANT));
if already {
return Ok(());
}
let start = grammar.start.clone();
let start_rule = grammar
.rules
.iter_mut()
.find(|r| matches!(r, Peg::Rule { name, .. } if *name == start));
let Some(Peg::Rule { body, .. }) = start_rule else {
return Err(BuildError::single(Diagnostic::error(
peg::codes::UNKNOWN_RULE,
format!("cannot add import syntax: start rule `{start}` is not defined"),
)));
};
match body.as_mut() {
Peg::Choice { alts, .. } => alts.push(peg::rule_ref(ids, IMPORT_VARIANT)),
_ => {
let dummy = peg::token(ids, "");
let old = std::mem::replace(body.as_mut(), dummy);
**body = peg::choice(ids, vec![old, peg::rule_ref(ids, IMPORT_VARIANT)]);
}
}
grammar.rules.push(peg::rule(
ids,
IMPORT_VARIANT,
peg::node(
ids,
IMPORT_VARIANT,
peg::seq(
ids,
vec![
peg::token(ids, "%kw:@import"),
peg::field(ids, IMPORT_SPEC_FIELD, peg::token(ids, "%str")),
],
),
),
));
Ok(())
}
struct LoadState<'a> {
resolver: &'a mut dyn SourceResolver,
limits: &'a ImportLimits,
schema: &'a NodeSchema,
grammar: Option<&'a Grammar>,
cache: BTreeMap<SourceId, CacheEntry>,
stack: Vec<SourceId>,
total_bytes: usize,
}
enum CacheEntry {
Pending,
Ready(ParseTree),
Failed(Vec<Diagnostic>),
}
impl LoadState<'_> {
fn chain(&self, tail: Option<&SourceId>) -> String {
let mut parts = vec!["<root>".to_string()];
parts.extend(self.stack.iter().map(SourceId::to_string));
if let Some(t) = tail {
parts.push(t.to_string());
}
parts.join(" → ")
}
}
fn expand(
mut tree: ParseTree,
importer: Option<&SourceId>,
depth: usize,
state: &mut LoadState<'_>,
) -> Result<ParseTree, Vec<Diagnostic>> {
if tree.variant == IMPORT_VARIANT {
let spec = placeholder_spec(&tree)?;
return resolve_import(&spec, importer, depth, state);
}
let mut diags = Vec::new();
for (_, slot) in &mut tree.children {
for child in std::mem::take(slot) {
match expand(child, importer, depth, state) {
Ok(t) => slot.push(t),
Err(ds) => diags.extend(ds),
}
}
}
for (_, entries) in &mut tree.keyed_children {
for (key, child) in std::mem::take(entries) {
match expand(child, importer, depth, state) {
Ok(t) => entries.push((key, t)),
Err(ds) => diags.extend(ds),
}
}
}
if diags.is_empty() {
Ok(tree)
} else {
Err(diags)
}
}
fn placeholder_spec(tree: &ParseTree) -> Result<String, Vec<Diagnostic>> {
match tree.field(IMPORT_SPEC_FIELD) {
Some(RawValue::Json(Value::String(s))) => Ok(s.clone()),
Some(RawValue::Text(s)) => Ok(s.clone()),
_ => Err(vec![
Diagnostic::error(
import_codes::SPEC_SHAPE,
format!(
"`{IMPORT_VARIANT}` placeholder lacks a string `{IMPORT_SPEC_FIELD}` payload",
),
)
.with_span(tree.span),
]),
}
}
fn resolve_import(
spec: &str,
importer: Option<&SourceId>,
depth: usize,
state: &mut LoadState<'_>,
) -> Result<ParseTree, Vec<Diagnostic>> {
if depth + 1 > state.limits.max_depth {
return Err(vec![Diagnostic::error(
import_codes::DEPTH_EXCEEDED,
format!(
"import nesting exceeds max_depth {} at `{spec}` (chain: {})",
state.limits.max_depth,
state.chain(None),
),
)]);
}
let id = state.resolver.resolve(importer, spec).map_err(|e| {
vec![Diagnostic::error(
import_codes::RESOLVE_FAILED,
format!(
"cannot resolve import `{spec}`: {e} (chain: {})",
state.chain(None)
),
)]
})?;
match state.cache.get(&id) {
Some(CacheEntry::Pending) => {
return Err(vec![Diagnostic::error(
import_codes::CYCLE,
format!("import cycle detected: {}", state.chain(Some(&id))),
)]);
}
Some(CacheEntry::Ready(tree)) => return Ok(tree.clone()),
Some(CacheEntry::Failed(diags)) => return Err(diags.clone()),
None => {}
}
if state.cache.len() + 1 > state.limits.max_sources {
return Err(vec![Diagnostic::error(
import_codes::SOURCE_LIMIT,
format!(
"import expansion exceeds max_sources {} at `{id}` (chain: {})",
state.limits.max_sources,
state.chain(None),
),
)]);
}
let fetched = match state.resolver.fetch(&id) {
Ok(s) => s,
Err(e) => {
let diags = vec![Diagnostic::error(
import_codes::FETCH_FAILED,
format!(
"cannot fetch import `{id}`: {e} (chain: {})",
state.chain(None)
),
)];
state.cache.insert(id, CacheEntry::Failed(diags.clone()));
return Err(diags);
}
};
let source_len = match &fetched {
ImportSource::Json(text) | ImportSource::Text(text) => text.len(),
};
state.total_bytes += source_len;
if state.total_bytes > state.limits.max_total_bytes {
let diags = vec![Diagnostic::error(
import_codes::BYTE_LIMIT,
format!(
"import expansion exceeds max_total_bytes {} at `{id}` (chain: {})",
state.limits.max_total_bytes,
state.chain(None),
),
)];
state.cache.insert(id, CacheEntry::Failed(diags.clone()));
return Err(diags);
}
let parsed = match fetched {
ImportSource::Json(text) => serde_bridge::from_json_str(&text, state.schema),
ImportSource::Text(text) => match state.grammar {
Some(grammar) => grammar.parse(&text),
None => {
let diags = vec![Diagnostic::error(
import_codes::TEXT_UNSUPPORTED,
format!(
"import `{id}` is a text source but this Loader has no \
grammar — configure one with `Loader::with_grammar` \
(chain: {})",
state.chain(None),
),
)];
state.cache.insert(id, CacheEntry::Failed(diags.clone()));
return Err(diags);
}
},
};
let parsed = match parsed {
Ok(tree) => tree,
Err(err) => {
let diags = in_import(&id, state, err.diagnostics);
state.cache.insert(id, CacheEntry::Failed(diags.clone()));
return Err(diags);
}
};
state.cache.insert(id.clone(), CacheEntry::Pending);
state.stack.push(id.clone());
let expanded = expand(parsed, Some(&id), depth + 1, state);
state.stack.pop();
match expanded {
Ok(tree) => {
state.cache.insert(id, CacheEntry::Ready(tree.clone()));
Ok(tree)
}
Err(diags) => {
let diags = in_import(&id, state, diags);
state.cache.insert(id, CacheEntry::Failed(diags.clone()));
Err(diags)
}
}
}
fn in_import(id: &SourceId, state: &LoadState<'_>, diags: Vec<Diagnostic>) -> Vec<Diagnostic> {
let marker = format!("in import `{id}`");
if diags
.first()
.is_some_and(|d| d.code == import_codes::IN_IMPORT && d.message.starts_with(&marker))
{
return diags;
}
let mut out = vec![Diagnostic::error(
import_codes::IN_IMPORT,
format!("{marker} (chain: {})", state.chain(Some(id))),
)];
out.extend(diags);
out
}