use bitmask_enum::bitmask;
use compact_str::CompactString;
use fnv::FnvBuildHasher;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::Duration;
use web_time::Instant;
use crate::TraverserValue;
use crate::error::{TraverserError, TraverserResult};
use crate::graph::{ExecutionPolicy, GraphStorage};
use crate::traversal::memory_budget::{
MEMORY_HEADROOM, MEMORY_LIMIT, MEMORY_LIMIT_AUTO, MemoryBudget,
};
use crate::traversal::optimizer::path_analysis::PathMode;
use crate::traversal::side_effect::SideEffectValue;
use crate::traversal::step::ProfileContext;
use crate::traversal::step::flatbarrier_steps::step_dedup::DedupKey;
use crate::value::{ElementProperty, PathArena, PathRef};
#[bitmask(u32)]
pub(crate) enum TraversalStrategy {
ProfileEnabled,
Diagnostics,
}
#[bitmask(u32)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum ProfileType {
Memory,
}
impl Default for ProfileType {
fn default() -> Self {
Self::none()
}
}
pub(crate) struct TraverserContext<'a, S: GraphStorage> {
pub(crate) storage: &'a mut S,
pub(crate) strategy: TraversalStrategy,
pub(crate) side_effect: HashMap<CompactString, SideEffectValue<S>>,
pub(crate) profile: ProfileContext,
pub(crate) profile_type: ProfileType,
pub(crate) paths: PathArena<S>,
pub(crate) sacks: crate::value::SackArena,
pub(crate) initial_sack: crate::value::SackRef,
pub(crate) path_mode: PathMode,
pub(crate) scratch_path: Vec<(u64, PathRef)>,
pub(crate) glob_path:
crate::traversal::step::flatmap_steps::step_glob_path::GlobPathScratch<S::VertexId>,
pub(crate) options: ExecutionOptions,
pub(crate) deadline: Option<Deadline>,
pub(crate) loop_stack: Vec<LoopFrame<S>>,
pub(crate) loop_persist: bool,
pub(crate) bulk_merge: bool,
pub(crate) bulk_active: bool,
pub(crate) sack_merge: Option<crate::value::Operator>,
pub(crate) bulk_one: bool,
pub(crate) auto_merge: bool,
rng: Option<rand::rngs::StdRng>,
}
pub(crate) struct LoopFrame<S: GraphStorage> {
pub(crate) name: Option<Arc<str>>,
pub(crate) loops: u32,
pub(crate) dedup_seen: HashMap<usize, HashSet<DedupKey<S>, FnvBuildHasher>>,
}
impl<S: GraphStorage> LoopFrame<S> {
pub(crate) fn new(name: Option<Arc<str>>) -> Self {
Self {
name,
loops: 0,
dedup_seen: HashMap::new(),
}
}
}
impl<S: GraphStorage> std::fmt::Debug for LoopFrame<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LoopFrame")
.field("name", &self.name)
.field("loops", &self.loops)
.field("dedup_sets", &self.dedup_seen.len())
.finish()
}
}
impl<'a, S: GraphStorage> TraverserContext<'a, S> {
#[inline]
pub(crate) fn with_persist<R>(&mut self, persist: bool, run: impl FnOnce(&mut Self) -> R) -> R {
let outer = std::mem::replace(&mut self.loop_persist, persist);
let result = run(self);
self.loop_persist = outer;
result
}
pub(crate) fn new(
storage: &'a mut S,
strategy: TraversalStrategy,
side_effect: HashMap<CompactString, SideEffectValue<S>>,
) -> Self {
Self {
storage,
strategy,
side_effect,
profile: ProfileContext::new(),
profile_type: ProfileType::none(),
paths: PathArena::new(),
sacks: crate::value::SackArena::new(),
initial_sack: crate::value::SackRef::NONE,
path_mode: PathMode::None,
scratch_path: Vec::new(),
glob_path: Default::default(),
options: ExecutionOptions::default(),
deadline: None,
loop_stack: Vec::new(),
loop_persist: false,
bulk_merge: false,
bulk_active: false,
sack_merge: None,
bulk_one: false,
auto_merge: false,
rng: None,
}
}
pub(crate) fn with_options(
storage: &'a mut S,
strategy: TraversalStrategy,
side_effect: HashMap<CompactString, SideEffectValue<S>>,
options: ExecutionOptions,
) -> Self {
let mut context = Self::new(storage, strategy, side_effect);
context.deadline = Deadline::start(
options.evaluation_timeout,
crate::traversal::cancel::CancelToken::current(),
);
context.options = options;
context
}
pub(crate) fn rng(&mut self) -> &mut rand::rngs::StdRng {
use rand::SeedableRng;
let seed = self.options.random_seed;
self.rng.get_or_insert_with(|| match seed {
Some(seed) => rand::rngs::StdRng::seed_from_u64(seed),
None => rand::rngs::StdRng::from_entropy(),
})
}
#[inline]
pub(crate) fn check_deadline(&self) -> TraverserResult<()> {
match &self.deadline {
None => Ok(()),
Some(deadline) => deadline.check(),
}
}
#[inline]
pub(crate) fn check_traverser_limit(&self, alive: usize) -> TraverserResult<()> {
let alive = alive as u64;
if alive > self.options.max_traversers {
return Err(Box::new(TraverserError::ResourceLimitExceeded {
limit: MAX_TRAVERSERS.to_string(),
max: Some(self.options.max_traversers),
observed: Some(alive),
}));
}
Ok(())
}
#[inline]
pub(crate) fn check_memory_budget(&self, alive: usize) -> TraverserResult<()> {
match &self.options.memory {
None => Ok(()),
Some(budget) => budget.check_boundary(&*self.storage, alive),
}
}
#[inline]
pub(crate) fn check_value_depth(&self, value: &TraverserValue<S>) -> TraverserResult<()> {
if crate::value::depth::traverser_value_exceeds_depth(value, self.options.max_value_depth) {
return Err(self.value_depth_error());
}
Ok(())
}
#[inline]
pub(crate) fn check_traverser_depth(
&self,
traverser: &crate::Traverser<S>,
) -> TraverserResult<()> {
self.check_value_depth(&traverser.value)?;
if !traverser.sack.is_none() {
self.check_property_depth(self.sacks.get(traverser.sack))?;
}
Ok(())
}
#[inline]
pub(crate) fn check_loop_traverser(
&self,
traverser: &crate::Traverser<S>,
) -> TraverserResult<()> {
self.check_traverser_depth(traverser)?;
let max = self.options.max_materialized_bytes;
let mut bytes =
crate::traversal::step::profile::estimate_traverser_value_bytes(&traverser.value)
.saturating_add(self.paths.value_bytes());
if !traverser.sack.is_none() {
bytes = bytes.saturating_add(self.sacks.get(traverser.sack).estimated_heap_bytes());
}
if bytes > max {
return Err(crate::traversal::bulk::materialized_bytes_error(max, bytes));
}
if let Some(budget) = &self.options.memory {
budget.check(bytes)?;
}
Ok(())
}
#[inline]
pub(crate) fn check_string_bytes(&self, len: usize) -> TraverserResult<()> {
check_string_bytes(len, self.options.max_string_bytes)
}
#[inline]
pub(crate) fn check_property_depth(&self, value: &ElementProperty) -> TraverserResult<()> {
if crate::value::depth::property_exceeds_depth(value, self.options.max_value_depth) {
return Err(self.value_depth_error());
}
Ok(())
}
#[inline]
pub(crate) fn check_written_depth(
&self,
key: &crate::value::PropKey,
value: &ElementProperty,
) -> TraverserResult<()> {
let below = match key {
crate::value::PropKey::Name(_) => 0,
crate::value::PropKey::Path(path) => path.segments().len().saturating_sub(1),
};
let max = self.options.max_value_depth;
if below > max || crate::value::depth::property_exceeds_depth(value, max - below) {
return Err(self.value_depth_error());
}
Ok(())
}
#[cold]
pub(crate) fn value_depth_error(&self) -> Box<TraverserError> {
Box::new(TraverserError::ResourceLimitExceeded {
limit: MAX_VALUE_DEPTH.to_string(),
max: Some(self.options.max_value_depth as u64),
observed: None,
})
}
#[inline]
pub(crate) fn tick(&mut self) -> TraverserResult<()> {
tick_deadline(&mut self.deadline)
}
#[inline]
pub(crate) fn tick_n(&mut self, n: usize) -> TraverserResult<()> {
match &mut self.deadline {
None => Ok(()),
Some(deadline) => deadline.tick_n(n),
}
}
}
#[inline]
pub(crate) fn check_string_bytes(len: usize, max: u64) -> TraverserResult<()> {
if len as u64 > max {
return Err(string_bytes_error(max, len));
}
Ok(())
}
#[cold]
fn string_bytes_error(max: u64, len: usize) -> Box<TraverserError> {
Box::new(TraverserError::ResourceLimitExceeded {
limit: MAX_STRING_BYTES.to_string(),
max: Some(max),
observed: Some(len as u64),
})
}
#[inline]
pub(crate) fn tick_deadline(deadline: &mut Option<Deadline>) -> TraverserResult<()> {
match deadline {
None => Ok(()),
Some(deadline) => deadline.tick(),
}
}
pub(crate) const EVALUATION_TIMEOUT: &str = "evaluationTimeout";
pub(crate) const REPEAT_ORDER: &str = "repeat.order";
pub(crate) const REPEAT_MAX_LOOPS: &str = "repeat.max_loops";
pub(crate) const DEFAULT_REPEAT_MAX_LOOPS: u32 = 10_000;
pub(crate) const BULK_MERGE: &str = "bulk.merge";
pub(crate) const BULK_ONE: &str = "bulk.one";
pub(crate) const BULK_MAX_EXPANSION: &str = "bulk.max_expansion";
pub(crate) const DEFAULT_BULK_MAX_EXPANSION: u64 = 10_000_000;
pub(crate) const MAX_MATERIALIZED_BYTES: &str = "traversal.max_materialized_bytes";
#[cfg(not(target_arch = "wasm32"))]
pub(crate) const DEFAULT_MAX_MATERIALIZED_BYTES: u64 = 1024 * 1024 * 1024;
#[cfg(target_arch = "wasm32")]
pub(crate) const DEFAULT_MAX_MATERIALIZED_BYTES: u64 = 256 * 1024 * 1024;
pub(crate) const MAX_STRING_BYTES: &str = "traversal.max_string_bytes";
pub(crate) const DEFAULT_MAX_STRING_BYTES: u64 = 16 * 1024 * 1024;
pub(crate) const MAX_TRAVERSERS: &str = "traversal.max_traversers";
pub(crate) const DEFAULT_MAX_TRAVERSERS: u64 = 10_000_000;
pub(crate) const MAX_VALUE_DEPTH: &str = "traversal.max_value_depth";
pub fn traversal_limit_defaults() -> Vec<(&'static str, ElementProperty)> {
let int = |n: u64| ElementProperty::from(i64::try_from(n).unwrap_or(i64::MAX));
vec![
(MAX_TRAVERSERS, int(DEFAULT_MAX_TRAVERSERS)),
(MAX_MATERIALIZED_BYTES, int(DEFAULT_MAX_MATERIALIZED_BYTES)),
(MAX_STRING_BYTES, int(DEFAULT_MAX_STRING_BYTES)),
(
MAX_VALUE_DEPTH,
int(crate::value::depth::DEFAULT_MAX_VALUE_DEPTH as u64),
),
(BULK_MAX_EXPANSION, int(DEFAULT_BULK_MAX_EXPANSION)),
(REPEAT_MAX_LOOPS, int(u64::from(DEFAULT_REPEAT_MAX_LOOPS))),
]
}
pub(crate) const OPTIMIZER_DISABLED: &str = "optimizer.disabled";
pub(crate) const OPTIMIZER_ENABLED: &str = "optimizer.enabled";
pub(crate) const OPTIMIZER_RULE_ALL: &str = "all";
pub(crate) const RANDOM_SEED: &str = "random.seed";
pub(crate) const PATH_ANALYSIS: &str = "path.analysis";
pub(crate) const RENDER_SPELLING: &str = "render.spelling";
#[allow(dead_code)]
pub(crate) const KNOWN_OPTION_KEYS: &[&str] = &[
EVALUATION_TIMEOUT,
REPEAT_ORDER,
REPEAT_MAX_LOOPS,
OPTIMIZER_DISABLED,
OPTIMIZER_ENABLED,
PATH_ANALYSIS,
BULK_MERGE,
BULK_ONE,
BULK_MAX_EXPANSION,
MAX_TRAVERSERS,
RANDOM_SEED,
RENDER_SPELLING,
MAX_VALUE_DEPTH,
MAX_MATERIALIZED_BYTES,
MAX_STRING_BYTES,
MEMORY_LIMIT,
MEMORY_HEADROOM,
];
pub(crate) const ADMIN_OPTION_KEYS: &[&str] = &[
EVALUATION_TIMEOUT,
REPEAT_MAX_LOOPS,
BULK_MAX_EXPANSION,
MAX_TRAVERSERS,
MAX_VALUE_DEPTH,
MAX_MATERIALIZED_BYTES,
MAX_STRING_BYTES,
MEMORY_LIMIT,
MEMORY_HEADROOM,
OPTIMIZER_DISABLED,
OPTIMIZER_ENABLED,
PATH_ANALYSIS,
BULK_MERGE,
];
#[allow(dead_code)]
pub(crate) const HARMLESS_OPTION_KEYS: &[&str] =
&[REPEAT_ORDER, BULK_ONE, RANDOM_SEED, RENDER_SPELLING];
pub(crate) fn is_admin_option(key: &str) -> bool {
ADMIN_OPTION_KEYS.contains(&key)
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum RepeatOrder {
#[default]
Bfs,
Dfs,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ExecutionOptions {
pub(crate) evaluation_timeout: Option<Duration>,
pub(crate) repeat_order: RepeatOrder,
pub(crate) repeat_max_loops: u32,
pub(crate) optimizer_disabled: HashSet<String>,
pub(crate) optimizer_enabled: HashSet<String>,
pub(crate) path_analysis: bool,
pub(crate) bulk_merge: bool,
pub(crate) bulk_one: bool,
pub(crate) bulk_max_expansion: u64,
pub(crate) max_traversers: u64,
pub(crate) random_seed: Option<u64>,
pub(crate) render_spelling: crate::traversal::Spelling,
pub(crate) max_value_depth: usize,
pub(crate) max_materialized_bytes: u64,
pub(crate) max_string_bytes: u64,
pub(crate) memory: Option<MemoryBudget>,
}
impl Default for ExecutionOptions {
fn default() -> Self {
Self {
evaluation_timeout: None,
repeat_order: RepeatOrder::default(),
repeat_max_loops: DEFAULT_REPEAT_MAX_LOOPS,
optimizer_disabled: HashSet::new(),
optimizer_enabled: HashSet::new(),
path_analysis: true,
bulk_merge: true,
bulk_one: false,
bulk_max_expansion: DEFAULT_BULK_MAX_EXPANSION,
max_traversers: DEFAULT_MAX_TRAVERSERS,
random_seed: None,
render_spelling: crate::traversal::Spelling::Snake,
max_value_depth: crate::value::depth::DEFAULT_MAX_VALUE_DEPTH,
max_materialized_bytes: DEFAULT_MAX_MATERIALIZED_BYTES,
max_string_bytes: DEFAULT_MAX_STRING_BYTES,
memory: None,
}
}
}
impl ExecutionOptions {
pub(crate) fn parse(
options: &HashMap<CompactString, ElementProperty>,
policy: &ExecutionPolicy,
) -> TraverserResult<Self> {
let mut parsed = Self::default();
if let Some(value) = resolve_option(EVALUATION_TIMEOUT, options, policy)? {
parsed.evaluation_timeout = parse_timeout(value)?;
}
if let Some(value) = resolve_option(REPEAT_ORDER, options, policy)? {
parsed.repeat_order = parse_repeat_order(value)?;
}
if let Some(value) = resolve_option(REPEAT_MAX_LOOPS, options, policy)? {
parsed.repeat_max_loops = parse_max_loops(value)?;
}
if let Some(value) = resolve_option(OPTIMIZER_DISABLED, options, policy)? {
parsed.optimizer_disabled = parse_rule_names(OPTIMIZER_DISABLED, value)?;
}
if let Some(value) = resolve_option(OPTIMIZER_ENABLED, options, policy)? {
parsed.optimizer_enabled = parse_rule_names(OPTIMIZER_ENABLED, value)?;
}
if let Some(value) = resolve_option(PATH_ANALYSIS, options, policy)? {
parsed.path_analysis = value.as_boolean().unwrap_or(true);
}
if let Some(value) = resolve_option(BULK_MERGE, options, policy)? {
parsed.bulk_merge = value.as_boolean().ok_or_else(|| {
Box::new(TraverserError::InvalidOption {
key: BULK_MERGE.to_string(),
reason: format!(
"expected true or false, got {} {}",
property_kind(value),
crate::value::literal::rhai_literal(value)
),
})
})?;
}
if let Some(value) = resolve_option(BULK_ONE, options, policy)? {
parsed.bulk_one = value.as_boolean().ok_or_else(|| {
Box::new(TraverserError::InvalidOption {
key: BULK_ONE.to_string(),
reason: format!(
"expected true or false, got {} {}",
property_kind(value),
crate::value::literal::rhai_literal(value)
),
})
})?;
}
if let Some(value) = resolve_option(BULK_MAX_EXPANSION, options, policy)? {
parsed.bulk_max_expansion = parse_max_expansion(value)?;
}
if let Some(value) = resolve_option(MAX_TRAVERSERS, options, policy)? {
parsed.max_traversers = parse_positive_count(MAX_TRAVERSERS, value, "traversers")?;
}
if let Some(value) = resolve_option(RANDOM_SEED, options, policy)? {
parsed.random_seed = Some(parse_random_seed(value)?);
}
if let Some(value) = resolve_option(RENDER_SPELLING, options, policy)? {
parsed.render_spelling = parse_spelling(value)?;
}
if let Some(value) = resolve_option(MAX_VALUE_DEPTH, options, policy)? {
parsed.max_value_depth =
usize::try_from(parse_positive_count(MAX_VALUE_DEPTH, value, "levels")?)
.unwrap_or(usize::MAX);
}
if let Some(value) = resolve_option(MAX_MATERIALIZED_BYTES, options, policy)? {
parsed.max_materialized_bytes =
parse_positive_count(MAX_MATERIALIZED_BYTES, value, "bytes")?;
}
if let Some(value) = resolve_option(MAX_STRING_BYTES, options, policy)? {
parsed.max_string_bytes = parse_positive_count(MAX_STRING_BYTES, value, "bytes")?;
}
parsed.memory = resolve_memory_limit(options, policy)?.map(MemoryBudget::new);
if let Some(name) = parsed
.optimizer_disabled
.intersection(&parsed.optimizer_enabled)
.next()
{
return Err(Box::new(TraverserError::InvalidOption {
key: OPTIMIZER_ENABLED.to_string(),
reason: format!(
"\"{name}\" is in both '{OPTIMIZER_ENABLED}' and '{OPTIMIZER_DISABLED}'; a \
rule cannot be both enabled and disabled for the same execution"
),
}));
}
Ok(parsed)
}
}
pub(crate) fn resolve_memory_limit(
options: &HashMap<CompactString, ElementProperty>,
policy: &ExecutionPolicy,
) -> TraverserResult<Option<u64>> {
let headroom = match resolve_option(MEMORY_HEADROOM, options, policy)? {
Some(value) => Some(parse_headroom(value)?),
None => None,
};
match resolve_option(MEMORY_LIMIT, options, policy)? {
Some(value) => parse_memory_limit(value, headroom),
None => Ok(None),
}
}
fn resolve_option<'q>(
key: &'static str,
options: &'q HashMap<CompactString, ElementProperty>,
policy: &'q ExecutionPolicy,
) -> TraverserResult<Option<&'q ElementProperty>> {
if policy.is_locked(key) {
if options.contains_key(key) {
return Err(Box::new(TraverserError::LockedOption {
key: key.to_string(),
}));
}
return Ok(policy.default_for(key));
}
match options.get(key) {
Some(value) => Ok(Some(value)),
None => Ok(policy.default_for(key)),
}
}
fn parse_rule_names(
key: &'static str,
value: &ElementProperty,
) -> TraverserResult<HashSet<String>> {
let invalid = |reason: String| {
Box::new(TraverserError::InvalidOption {
key: key.to_string(),
reason,
})
};
let items = value.as_array().ok_or_else(|| {
invalid(format!(
"expected an array of optimizer rule names, got {} {}",
property_kind(value),
crate::value::literal::rhai_literal(value)
))
})?;
let mut names = HashSet::with_capacity(items.len());
for item in items {
let name = item.as_str().ok_or_else(|| {
invalid(format!(
"expected an array of optimizer rule names (strings), got {} {} in the list",
property_kind(item),
crate::value::literal::rhai_literal(item)
))
})?;
if name == OPTIMIZER_RULE_ALL {
names.extend(
crate::traversal::optimizer::KNOWN_RULES
.iter()
.map(|(known, _)| known.to_string()),
);
continue;
}
if !crate::traversal::optimizer::KNOWN_RULES
.iter()
.any(|(known, _)| *known == name)
{
let valid: Vec<&str> = crate::traversal::optimizer::KNOWN_RULES
.iter()
.map(|(known, _)| *known)
.chain(std::iter::once(OPTIMIZER_RULE_ALL))
.collect();
return Err(invalid(format!(
"unknown optimizer rule \"{name}\"; valid names are: {}",
valid.join(", ")
)));
}
names.insert(name.to_string());
}
Ok(names)
}
fn parse_repeat_order(value: &ElementProperty) -> TraverserResult<RepeatOrder> {
match value.as_str() {
Some(order) if order.eq_ignore_ascii_case("bfs") => Ok(RepeatOrder::Bfs),
Some(order) if order.eq_ignore_ascii_case("dfs") => Ok(RepeatOrder::Dfs),
Some(order) => Err(Box::new(TraverserError::InvalidOption {
key: REPEAT_ORDER.to_string(),
reason: format!("expected \"bfs\" or \"dfs\", got \"{order}\""),
})),
None => Err(Box::new(TraverserError::InvalidOption {
key: REPEAT_ORDER.to_string(),
reason: format!(
"expected \"bfs\" or \"dfs\", got {} {}",
property_kind(value),
crate::value::literal::rhai_literal(value)
),
})),
}
}
fn parse_positive_count(
key: &'static str,
value: &ElementProperty,
unit: &str,
) -> TraverserResult<u64> {
let invalid = |reason: String| {
Box::new(TraverserError::InvalidOption {
key: key.to_string(),
reason,
})
};
let limit = match value {
ElementProperty::Number(number) => match (number.as_i64(), number.as_f64()) {
(Some(n), _) => n,
(None, Some(n)) if n.is_finite() && n.fract() == 0.0 && n.abs() <= i64::MAX as f64 => {
n as i64
}
_ => {
return Err(invalid(format!(
"expected a positive integer number of {unit}, got {number}"
)));
}
},
other => {
return Err(invalid(format!(
"expected a positive integer number of {unit}, got {} {}",
property_kind(other),
crate::value::literal::rhai_literal(other)
)));
}
};
match u64::try_from(limit) {
Ok(limit) if limit >= 1 => Ok(limit),
_ => Err(invalid(format!(
"expected a positive integer number of {unit}, got {limit}"
))),
}
}
pub(crate) fn requested_spelling(
options: &HashMap<CompactString, ElementProperty>,
) -> crate::traversal::Spelling {
options
.get(RENDER_SPELLING)
.and_then(ElementProperty::as_str)
.and_then(crate::traversal::Spelling::parse)
.unwrap_or_default()
}
fn parse_spelling(value: &ElementProperty) -> TraverserResult<crate::traversal::Spelling> {
value
.as_str()
.and_then(crate::traversal::Spelling::parse)
.ok_or_else(|| {
Box::new(TraverserError::InvalidOption {
key: RENDER_SPELLING.to_string(),
reason: format!(
"expected \"snake\" or \"camel\", got {} {}",
property_kind(value),
crate::value::literal::rhai_literal(value)
),
})
})
}
fn parse_random_seed(value: &ElementProperty) -> TraverserResult<u64> {
match value {
ElementProperty::Number(number) => match number.as_i64() {
Some(seed) => Ok(seed as u64),
None => Err(Box::new(TraverserError::InvalidOption {
key: RANDOM_SEED.to_string(),
reason: format!("expected an integer seed, got {number}"),
})),
},
other => Err(Box::new(TraverserError::InvalidOption {
key: RANDOM_SEED.to_string(),
reason: format!(
"expected an integer seed, got {} {}",
property_kind(other),
crate::value::literal::rhai_literal(other)
),
})),
}
}
fn parse_memory_limit(
value: &ElementProperty,
headroom: Option<u64>,
) -> TraverserResult<Option<u64>> {
if let Some(text) = value.as_str() {
if text.eq_ignore_ascii_case(MEMORY_LIMIT_AUTO) {
return Ok(crate::traversal::memory_budget::resolve_auto(headroom));
}
return Err(Box::new(TraverserError::InvalidOption {
key: MEMORY_LIMIT.to_string(),
reason: format!(
"expected a positive integer number of bytes or \"auto\", got \"{text}\""
),
}));
}
parse_positive_count(MEMORY_LIMIT, value, "bytes").map(Some)
}
fn parse_headroom(value: &ElementProperty) -> TraverserResult<u64> {
if matches!(value, ElementProperty::Number(number) if number.as_i64() == Some(0)) {
return Ok(0);
}
parse_positive_count(MEMORY_HEADROOM, value, "bytes")
}
fn parse_max_expansion(value: &ElementProperty) -> TraverserResult<u64> {
parse_positive_count(BULK_MAX_EXPANSION, value, "values")
}
fn parse_max_loops(value: &ElementProperty) -> TraverserResult<u32> {
let invalid = |reason: String| {
Box::new(TraverserError::InvalidOption {
key: REPEAT_MAX_LOOPS.to_string(),
reason,
})
};
let loops = match value {
ElementProperty::Number(number) => match (number.as_i64(), number.as_f64()) {
(Some(n), _) => n,
(None, Some(n)) if n.is_finite() && n.fract() == 0.0 && n.abs() <= i64::MAX as f64 => {
n as i64
}
_ => {
return Err(invalid(format!(
"expected a positive integer number of iterations, got {number}"
)));
}
},
other => {
return Err(invalid(format!(
"expected a positive integer number of iterations, got {} {}",
property_kind(other),
crate::value::literal::rhai_literal(other)
)));
}
};
match u32::try_from(loops) {
Ok(loops) if loops >= 1 => Ok(loops),
_ => Err(invalid(format!(
"expected a positive integer number of iterations up to {}, got {loops}",
u32::MAX
))),
}
}
fn parse_timeout(value: &ElementProperty) -> TraverserResult<Option<Duration>> {
let invalid = |reason: String| {
Box::new(TraverserError::InvalidOption {
key: EVALUATION_TIMEOUT.to_string(),
reason,
})
};
let millis = match value {
ElementProperty::Number(number) => match (number.as_i64(), number.as_f64()) {
(Some(ms), _) if ms >= 0 => ms as u64,
(Some(ms), _) => {
return Err(invalid(format!(
"expected a non-negative number of milliseconds, got {ms}"
)));
}
(None, Some(ms))
if ms.is_finite() && ms.fract() == 0.0 && ms >= 0.0 && ms <= u64::MAX as f64 =>
{
ms as u64
}
(None, Some(ms)) if ms < 0.0 => {
return Err(invalid(format!(
"expected a non-negative number of milliseconds, got {ms}"
)));
}
_ => {
return Err(invalid(format!(
"expected an integer number of milliseconds, got {number}"
)));
}
},
other => {
return Err(invalid(format!(
"expected an integer number of milliseconds, got {} {}",
property_kind(other),
crate::value::literal::rhai_literal(other)
)));
}
};
Ok((millis > 0).then(|| Duration::from_millis(millis)))
}
fn property_kind(value: &ElementProperty) -> &'static str {
match value {
ElementProperty::Null => "null",
ElementProperty::String(_) => "string",
ElementProperty::Boolean(_) => "boolean",
ElementProperty::Array(_) => "array",
ElementProperty::Object(_) | ElementProperty::Map(_) => "object",
_ => "value",
}
}
#[derive(Debug, Clone)]
pub(crate) struct Deadline {
started: Instant,
at: Option<Instant>,
limit: Duration,
cancel: Option<crate::CancelToken>,
ticks: u32,
}
impl Deadline {
#[cfg(test)]
pub(crate) fn new(limit: Duration) -> Option<Self> {
Self::start(Some(limit), None)
}
pub(crate) fn start(
limit: Option<Duration>,
cancel: Option<crate::CancelToken>,
) -> Option<Self> {
let started = Instant::now();
let at = limit.and_then(|limit| started.checked_add(limit));
if at.is_none() && cancel.is_none() {
return None;
}
Some(Self {
started,
at,
limit: limit.unwrap_or_default(),
cancel,
ticks: 0,
})
}
#[inline]
pub(crate) fn check(&self) -> TraverserResult<()> {
if let Some(cancel) = &self.cancel
&& cancel.is_cancelled()
{
return Err(Box::new(TraverserError::Cancelled {
elapsed_ms: saturating_millis(self.started.elapsed()),
}));
}
let Some(at) = self.at else {
return Ok(());
};
let now = Instant::now();
if now >= at {
return Err(Box::new(TraverserError::Timeout {
limit_ms: saturating_millis(self.limit),
elapsed_ms: saturating_millis(now.saturating_duration_since(self.started)),
}));
}
Ok(())
}
#[inline]
pub(crate) fn tick(&mut self) -> TraverserResult<()> {
self.ticks = self.ticks.wrapping_add(1);
if self.ticks & 1023 == 0 {
self.check()
} else {
Ok(())
}
}
#[inline]
pub(crate) fn tick_n(&mut self, n: usize) -> TraverserResult<()> {
let before = self.ticks;
self.ticks = before.wrapping_add(n as u32);
if n >= 1024 || (before >> 10) != (self.ticks >> 10) {
self.check()
} else {
Ok(())
}
}
}
fn saturating_millis(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::Number;
#[test]
fn traversal_limit_defaults_are_the_execution_defaults() {
let defaults: HashMap<&str, ElementProperty> =
traversal_limit_defaults().into_iter().collect();
let options = ExecutionOptions::default();
let int = |key: &str| match defaults.get(key) {
Some(ElementProperty::Number(n)) => n.as_i64().unwrap_or(-1),
other => panic!("{key}: {other:?}"),
};
assert_eq!(int(MAX_TRAVERSERS) as u64, options.max_traversers);
assert_eq!(
int(MAX_MATERIALIZED_BYTES) as u64,
options.max_materialized_bytes
);
assert_eq!(int(MAX_STRING_BYTES) as u64, options.max_string_bytes);
assert_eq!(int(MAX_VALUE_DEPTH) as usize, options.max_value_depth);
assert_eq!(int(BULK_MAX_EXPANSION) as u64, options.bulk_max_expansion);
assert_eq!(int(REPEAT_MAX_LOOPS) as u32, options.repeat_max_loops);
assert_eq!(defaults.len(), 6);
}
fn options(value: ElementProperty) -> HashMap<CompactString, ElementProperty> {
HashMap::from([(CompactString::new(EVALUATION_TIMEOUT), value)])
}
#[test]
fn evaluation_timeout_parses_integers_and_zero() {
let permissive = ExecutionPolicy::permissive();
let parsed = ExecutionOptions::parse(
&options(ElementProperty::Number(Number::from_i64(250))),
&permissive,
)
.unwrap();
assert_eq!(parsed.evaluation_timeout, Some(Duration::from_millis(250)));
let parsed = ExecutionOptions::parse(
&options(ElementProperty::Number(Number::from_f64(250.0))),
&permissive,
)
.unwrap();
assert_eq!(parsed.evaluation_timeout, Some(Duration::from_millis(250)));
let parsed = ExecutionOptions::parse(
&options(ElementProperty::Number(Number::from_i64(0))),
&permissive,
)
.unwrap();
assert_eq!(parsed.evaluation_timeout, None);
assert_eq!(
ExecutionOptions::parse(&HashMap::new(), &permissive).unwrap(),
ExecutionOptions::default()
);
}
#[test]
fn max_traversers_defaults_parses_and_rejects_invalid_values() {
let defaults =
ExecutionOptions::parse(&HashMap::new(), &ExecutionPolicy::permissive()).unwrap();
assert_eq!(defaults.max_traversers, DEFAULT_MAX_TRAVERSERS);
let options = HashMap::from([(
CompactString::new(MAX_TRAVERSERS),
ElementProperty::Number(Number::from_i64(7)),
)]);
let parsed = ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap();
assert_eq!(parsed.max_traversers, 7);
for bad in [
ElementProperty::Number(Number::from_i64(0)),
ElementProperty::String("7".into()),
] {
let options = HashMap::from([(CompactString::new(MAX_TRAVERSERS), bad)]);
let err =
ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap_err();
assert!(
matches!(*err, TraverserError::InvalidOption { ref key, .. } if key == MAX_TRAVERSERS)
);
}
}
#[test]
fn evaluation_timeout_rejects_invalid_values() {
for value in [
ElementProperty::Number(Number::from_i64(-1)),
ElementProperty::Number(Number::from_f64(1.5)),
ElementProperty::Number(Number::from_f64(-2.0)),
ElementProperty::String("abc".into()),
ElementProperty::Boolean(true),
ElementProperty::Null,
] {
let err = ExecutionOptions::parse(&options(value), &ExecutionPolicy::permissive())
.unwrap_err();
assert!(
matches!(*err, TraverserError::InvalidOption { ref key, .. } if key == EVALUATION_TIMEOUT)
);
}
}
#[test]
fn repeat_options_parse_and_default() {
let defaults =
ExecutionOptions::parse(&HashMap::new(), &ExecutionPolicy::permissive()).unwrap();
assert_eq!(defaults.repeat_order, RepeatOrder::Bfs);
assert_eq!(defaults.repeat_max_loops, DEFAULT_REPEAT_MAX_LOOPS);
let parsed = ExecutionOptions::parse(
&HashMap::from([
(
CompactString::new(REPEAT_ORDER),
ElementProperty::String("DFS".into()),
),
(
CompactString::new(REPEAT_MAX_LOOPS),
ElementProperty::Number(Number::from_i64(50)),
),
]),
&ExecutionPolicy::permissive(),
)
.unwrap();
assert_eq!(parsed.repeat_order, RepeatOrder::Dfs);
assert_eq!(parsed.repeat_max_loops, 50);
}
#[test]
fn repeat_options_reject_invalid_values() {
for (key, value) in [
(REPEAT_ORDER, ElementProperty::String("sideways".into())),
(REPEAT_ORDER, ElementProperty::Number(Number::from_i64(1))),
(
REPEAT_MAX_LOOPS,
ElementProperty::Number(Number::from_i64(0)),
),
(
REPEAT_MAX_LOOPS,
ElementProperty::Number(Number::from_i64(-3)),
),
(
REPEAT_MAX_LOOPS,
ElementProperty::Number(Number::from_i64(i64::from(u32::MAX) + 1)),
),
(
REPEAT_MAX_LOOPS,
ElementProperty::Number(Number::from_f64(1.5)),
),
(REPEAT_MAX_LOOPS, ElementProperty::String("10".into())),
] {
let options = HashMap::from([(CompactString::new(key), value)]);
let err =
ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap_err();
assert!(
matches!(*err, TraverserError::InvalidOption { key: ref k, .. } if k == key),
"{err}"
);
}
}
#[test]
fn bulk_options_parse_and_default() {
let defaults =
ExecutionOptions::parse(&HashMap::new(), &ExecutionPolicy::permissive()).unwrap();
assert!(defaults.bulk_merge);
assert_eq!(defaults.bulk_max_expansion, DEFAULT_BULK_MAX_EXPANSION);
let parsed = ExecutionOptions::parse(
&HashMap::from([
(
CompactString::new(BULK_MERGE),
ElementProperty::Boolean(false),
),
(
CompactString::new(BULK_MAX_EXPANSION),
ElementProperty::Number(Number::from_i64(5)),
),
]),
&ExecutionPolicy::permissive(),
)
.unwrap();
assert!(!parsed.bulk_merge);
assert_eq!(parsed.bulk_max_expansion, 5);
}
#[test]
fn bulk_options_reject_invalid_values() {
for (key, value) in [
(BULK_MERGE, ElementProperty::String("yes".into())),
(BULK_MERGE, ElementProperty::Number(Number::from_i64(1))),
(
BULK_MAX_EXPANSION,
ElementProperty::Number(Number::from_i64(0)),
),
(
BULK_MAX_EXPANSION,
ElementProperty::Number(Number::from_i64(-3)),
),
(
BULK_MAX_EXPANSION,
ElementProperty::Number(Number::from_f64(1.5)),
),
(BULK_MAX_EXPANSION, ElementProperty::String("10".into())),
(BULK_MAX_EXPANSION, ElementProperty::Boolean(true)),
] {
let options = HashMap::from([(CompactString::new(key), value)]);
let err =
ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap_err();
assert!(
matches!(*err, TraverserError::InvalidOption { key: ref k, .. } if k == key),
"{err}"
);
}
}
#[test]
fn byte_limits_default_parse_and_reject_invalid_values() {
let defaults =
ExecutionOptions::parse(&HashMap::new(), &ExecutionPolicy::permissive()).unwrap();
assert_eq!(
defaults.max_materialized_bytes,
DEFAULT_MAX_MATERIALIZED_BYTES
);
assert_eq!(defaults.max_string_bytes, 16 * 1024 * 1024);
assert_eq!(defaults.bulk_max_expansion, 10_000_000);
#[cfg(not(target_arch = "wasm32"))]
assert_eq!(DEFAULT_MAX_MATERIALIZED_BYTES, 1 << 30);
for key in [MAX_MATERIALIZED_BYTES, MAX_STRING_BYTES] {
let options =
HashMap::from([(CompactString::new(key), ElementProperty::Number(7.into()))]);
let parsed = ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap();
let value = if key == MAX_STRING_BYTES {
parsed.max_string_bytes
} else {
parsed.max_materialized_bytes
};
assert_eq!(value, 7, "{key}");
for bad in [
ElementProperty::Number(Number::from_i64(0)),
ElementProperty::Number(Number::from_f64(1.5)),
ElementProperty::String("1 GiB".into()),
] {
let options = HashMap::from([(CompactString::new(key), bad)]);
let err =
ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap_err();
assert!(
matches!(*err, TraverserError::InvalidOption { key: ref k, ref reason }
if k == key && reason.contains("number of bytes")),
"{err}"
);
}
assert!(is_admin_option(key));
}
}
#[test]
fn string_limit_check_compares_bytes() {
assert!(check_string_bytes(16, 16).is_ok());
let err = check_string_bytes(17, 16).unwrap_err();
assert!(matches!(
*err,
TraverserError::ResourceLimitExceeded { ref limit, max: Some(16), observed: Some(17) }
if limit == MAX_STRING_BYTES
));
}
#[test]
fn deadline_far_in_the_future_does_not_overflow() {
let _ = Deadline::new(Duration::from_millis(u64::MAX));
let mut deadline = Deadline::new(Duration::from_secs(3600)).unwrap();
for _ in 0..4096 {
deadline.tick().unwrap();
}
deadline.tick_n(5000).unwrap();
}
#[test]
fn expired_deadline_fails_on_the_next_clock_read() {
let mut deadline = Deadline::new(Duration::from_nanos(1)).unwrap();
std::thread::sleep(Duration::from_millis(2));
assert!(
matches!(deadline.check(), Err(ref e) if matches!(**e, TraverserError::Timeout { .. }))
);
let mut failed = false;
for _ in 0..1024 {
failed |= deadline.tick().is_err();
}
assert!(failed);
assert!(deadline.tick_n(1024).is_err());
}
fn rule_names(names: &[&str]) -> ElementProperty {
ElementProperty::Array(
names
.iter()
.map(|n| ElementProperty::String((*n).into()))
.collect(),
)
}
#[test]
fn optimizer_disabled_and_enabled_parse_known_rule_names() {
let options = HashMap::from([(
CompactString::new(OPTIMIZER_DISABLED),
rule_names(&["filter_reorder", "count_pushdown"]),
)]);
let parsed = ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap();
assert_eq!(
parsed.optimizer_disabled,
HashSet::from(["filter_reorder".to_string(), "count_pushdown".to_string()])
);
assert!(parsed.optimizer_enabled.is_empty());
}
#[test]
fn optimizer_disabled_rejects_unknown_rule_names() {
let options = HashMap::from([(
CompactString::new(OPTIMIZER_DISABLED),
rule_names(&["not_a_real_rule"]),
)]);
let err = ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap_err();
assert!(
matches!(*err, TraverserError::InvalidOption { ref key, .. } if key == OPTIMIZER_DISABLED)
);
}
#[test]
fn optimizer_disabled_all_expands_to_every_known_rule() {
let options =
HashMap::from([(CompactString::new(OPTIMIZER_DISABLED), rule_names(&["all"]))]);
let parsed = ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap();
let expected: HashSet<String> = crate::traversal::optimizer::KNOWN_RULES
.iter()
.map(|(name, _)| name.to_string())
.collect();
assert_eq!(parsed.optimizer_disabled, expected);
}
#[test]
fn optimizer_disabled_and_enabled_reject_a_shared_name() {
let options = HashMap::from([
(
CompactString::new(OPTIMIZER_DISABLED),
rule_names(&["filter_reorder"]),
),
(
CompactString::new(OPTIMIZER_ENABLED),
rule_names(&["filter_reorder"]),
),
]);
let err = ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap_err();
assert!(
matches!(*err, TraverserError::InvalidOption { ref key, .. } if key == OPTIMIZER_ENABLED)
);
}
#[test]
fn optimizer_disabled_rejects_a_non_array_value() {
let options = HashMap::from([(
CompactString::new(OPTIMIZER_DISABLED),
ElementProperty::String("filter_reorder".into()),
)]);
let err = ExecutionOptions::parse(&options, &ExecutionPolicy::permissive()).unwrap_err();
assert!(
matches!(*err, TraverserError::InvalidOption { ref key, .. } if key == OPTIMIZER_DISABLED)
);
}
#[test]
fn a_locked_key_the_query_tries_to_set_fails_with_locked_option() {
let policy = ExecutionPolicy::permissive()
.with_locked(EVALUATION_TIMEOUT)
.with_default(EVALUATION_TIMEOUT, 2000);
let options = options(ElementProperty::Number(Number::from_i64(999_999)));
let err = ExecutionOptions::parse(&options, &policy).unwrap_err();
assert!(
matches!(*err, TraverserError::LockedOption { ref key } if key == EVALUATION_TIMEOUT)
);
}
#[test]
fn a_locked_keys_policy_default_still_applies_when_the_query_omits_it() {
let policy = ExecutionPolicy::permissive()
.with_locked(EVALUATION_TIMEOUT)
.with_default(EVALUATION_TIMEOUT, 2000);
let parsed = ExecutionOptions::parse(&HashMap::new(), &policy).unwrap();
assert_eq!(parsed.evaluation_timeout, Some(Duration::from_millis(2000)));
}
#[test]
fn a_default_with_nothing_locked_can_still_be_overridden_by_the_query() {
let policy = ExecutionPolicy::permissive().with_default(EVALUATION_TIMEOUT, 2000);
let parsed = ExecutionOptions::parse(&HashMap::new(), &policy).unwrap();
assert_eq!(parsed.evaluation_timeout, Some(Duration::from_millis(2000)));
let parsed = ExecutionOptions::parse(
&options(ElementProperty::Number(Number::from_i64(9000))),
&policy,
)
.unwrap();
assert_eq!(parsed.evaluation_timeout, Some(Duration::from_millis(9000)));
}
#[test]
fn locking_repeat_max_loops_rejects_a_query_override_and_keeps_its_own_default() {
let policy = ExecutionPolicy::permissive()
.with_locked(REPEAT_MAX_LOOPS)
.with_default(REPEAT_MAX_LOOPS, 5);
let options = HashMap::from([(
CompactString::new(REPEAT_MAX_LOOPS),
ElementProperty::Number(Number::from_i64(999_999_999)),
)]);
let err = ExecutionOptions::parse(&options, &policy).unwrap_err();
assert!(
matches!(*err, TraverserError::LockedOption { ref key } if key == REPEAT_MAX_LOOPS)
);
let parsed = ExecutionOptions::parse(&HashMap::new(), &policy).unwrap();
assert_eq!(parsed.repeat_max_loops, 5);
}
#[test]
fn every_known_option_key_is_locked_through_the_same_generic_check() {
for key in KNOWN_OPTION_KEYS {
let policy = ExecutionPolicy::permissive().with_locked(*key);
let value = if *key == OPTIMIZER_DISABLED || *key == OPTIMIZER_ENABLED {
rule_names(&["filter_reorder"])
} else if *key == PATH_ANALYSIS || *key == BULK_MERGE || *key == BULK_ONE {
ElementProperty::Boolean(false)
} else if *key == REPEAT_ORDER {
ElementProperty::String("dfs".into())
} else {
ElementProperty::Number(Number::from_i64(42))
};
let options = HashMap::from([(CompactString::new(*key), value)]);
let err = ExecutionOptions::parse(&options, &policy).unwrap_err();
assert!(
matches!(*err, TraverserError::LockedOption { key: ref k } if k == key),
"key {key} did not reject a query override via the generic lock check: {err}"
);
}
}
#[test]
fn permissive_policy_matches_pre_policy_behavior() {
let permissive = ExecutionPolicy::permissive();
assert_eq!(
ExecutionOptions::parse(&HashMap::new(), &permissive).unwrap(),
ExecutionOptions::default()
);
let options = HashMap::from([(
CompactString::new(EVALUATION_TIMEOUT),
ElementProperty::Number(Number::from_i64(1234)),
)]);
let parsed = ExecutionOptions::parse(&options, &permissive).unwrap();
assert_eq!(parsed.evaluation_timeout, Some(Duration::from_millis(1234)));
}
}