use std::collections::HashMap;
use oneyaml_core::{Limits, Span, code};
use super::budget::{ReadShared, SharedRead};
use super::{Ev, Node, Source, Tag};
use crate::error::Error;
enum Taped<'a> {
Ev(Ev<'a>),
Alias {
target: usize,
span: Span,
},
}
#[derive(Clone, Copy)]
struct Anchor {
start: usize,
end: Option<usize>,
size: Size,
}
#[derive(Clone, Copy, Default, Debug, PartialEq, Eq)]
struct Size {
nodes: usize,
bytes: usize,
}
impl Size {
fn plus(self, o: Size) -> Size {
Size {
nodes: self.nodes.saturating_add(o.nodes),
bytes: self.bytes.saturating_add(o.bytes),
}
}
}
fn tag_len(tag: &Tag<'_>) -> usize {
match tag {
Tag::Named(t) => t.tag().len(),
_ => 0,
}
}
fn replay_cost(ev: &Ev<'_>) -> Size {
match &ev.node {
Node::Scalar { value, tag, .. } => Size {
nodes: 1,
bytes: value.len().saturating_add(tag_len(tag)),
},
Node::SeqStart { tag } | Node::MapStart { tag } => Size {
nodes: 1,
bytes: tag_len(tag),
},
_ => Size::default(),
}
}
fn own_bytes(ev: &Ev<'_>) -> usize {
match &ev.node {
Node::Scalar {
tag: Tag::Named(t), ..
}
| Node::SeqStart { tag: Tag::Named(t) }
| Node::MapStart { tag: Tag::Named(t) } => {
t.tag().shared_prefix().map_or(0, |p| p.text().len())
}
_ => 0,
}
}
#[cfg(test)]
thread_local! {
pub(crate) static REPLAYED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
struct Cursor {
start: usize,
pos: usize,
end: usize,
alias: Span,
alias_replayed: bool,
delivered: bool,
}
pub(crate) fn undefined_alias(name: &str, span: Span) -> Error {
let name = crate::error::segment(name);
Error::new(
code::UNDEFINED_ALIAS,
format!("alias `*{name}` refers to no anchor defined before it in this document"),
Some(span),
)
.with_help(format!("define `&{name}` on an earlier node"))
}
pub(crate) fn alias_budget_exceeded(budget: usize, own: usize, span: Option<Span>) -> Error {
Error::new(
code::ALIAS_BUDGET_EXCEEDED,
format!(
"aliases expand to more than {budget} nodes (the document has {own} nodes of its own)"
),
span,
)
.with_help(format!(
"raise Limits::alias_budget (currently {budget} for this document), \
or use Limits::unlimited() for fully trusted input"
))
}
pub(crate) fn recursive_alias(name: &str, span: Span) -> Error {
let name = crate::error::segment(name);
Error::new(
code::UNDEFINED_ALIAS,
format!("alias `*{name}` is inside the node anchored as `&{name}` (a recursive alias)"),
Some(span),
)
}
pub(crate) struct Expander<'a, S> {
src: S,
limits: Limits,
tape: Vec<Taped<'a>>,
anchors: Vec<Anchor>,
names: HashMap<&'a str, usize>,
recording: Vec<(usize, usize, Size)>,
replay: Vec<Cursor>,
src_depth: usize,
depth: usize,
own_nodes: usize,
expanded: usize,
peeked: Option<Ev<'a>>,
last_end: usize,
nodes_before: Option<usize>,
shared: SharedRead,
base: usize,
preloaded: usize,
}
impl<'a, S: Source<'a>> Expander<'a, S> {
pub(crate) fn new(src: S, limits: &Limits) -> Self {
Self {
src,
limits: limits.clone(),
tape: Vec::new(),
anchors: Vec::new(),
names: HashMap::new(),
recording: Vec::new(),
replay: Vec::new(),
src_depth: 0,
depth: 0,
own_nodes: 0,
expanded: 0,
peeked: None,
last_end: 0,
nodes_before: None,
shared: ReadShared::new(limits),
base: 0,
preloaded: 0,
}
}
pub(crate) fn shared(&self) -> SharedRead {
std::rc::Rc::clone(&self.shared)
}
pub(crate) fn source_mut(&mut self) -> &mut S {
&mut self.src
}
pub(crate) fn last_end(&self) -> usize {
self.last_end
}
pub(crate) fn starts_delivered_replay(&self) -> bool {
self.replay
.last()
.is_some_and(|c| c.delivered && c.pos == c.start.saturating_add(1))
}
pub(crate) fn peek(&mut self) -> Result<&Ev<'a>, Error> {
if self.peeked.is_none() {
self.peeked = Some(self.produce()?);
}
match &self.peeked {
Some(e) => Ok(e),
None => Err(Error::internal("peek without event", None)),
}
}
pub(crate) fn next(&mut self) -> Result<Ev<'a>, Error> {
let ev = match self.peeked.take() {
Some(e) => e,
None => self.produce()?,
};
self.last_end = ev.span.end;
Ok(ev)
}
fn produce(&mut self) -> Result<Ev<'a>, Error> {
loop {
let Some(top) = self.replay.last_mut() else {
return self.pull_source();
};
if top.pos >= top.end {
self.replay.pop();
continue;
}
let at = top.pos;
top.pos = at.saturating_add(1);
let edge = at == top.start || top.pos >= top.end;
let (alias, alias_replayed) = (top.alias, top.alias_replayed);
match self.tape.get(at) {
Some(Taped::Ev(e)) => {
let mut e = e.clone();
e.anchor = None;
e.replayed = true;
if edge {
e.span = alias;
e.replayed = alias_replayed;
}
#[cfg(test)]
REPLAYED.with(|r| r.set(r.get().saturating_add(1)));
if e.starts_node() {
self.count_expanded(e.span)?;
}
self.shared
.meter
.charge(replay_cost(&e).bytes, Some(e.span))?;
self.track_depth(&e)?;
return Ok(e);
}
Some(Taped::Alias { target, span }) => {
let (target, span) = (*target, *span);
self.push_replay(target, span, true)?;
}
None => return Err(Error::internal("replay past the tape", None)),
}
}
}
fn pull_source(&mut self) -> Result<Ev<'a>, Error> {
let mut ev = self.src.next_event()?;
if matches!(ev.node, Node::DocStart { .. }) {
self.tape.clear();
self.anchors.clear();
self.names.clear();
self.recording.clear();
self.own_nodes = 0;
self.expanded = 0;
self.nodes_before = None;
self.shared.meter.reset();
self.base = self.src.document_start().unwrap_or(ev.span.start);
for pre in self.src.take_preload()? {
self.preload(pre)?;
}
self.preloaded = self.tape.len();
}
self.shared
.meter
.read_to(ev.span.end.saturating_sub(self.base));
if ev.starts_node() {
self.own_nodes = self.own_nodes.saturating_add(1);
}
if let Node::Alias { name } = ev.node {
let target = self.resolve_alias(name, ev.span)?;
self.tape_alias(target, ev.span);
self.push_replay(target, ev.span, false)?;
return self.produce();
}
self.record(&ev);
self.shared.meter.charge(own_bytes(&ev), Some(ev.span))?;
ev.anchor = None;
self.track_depth(&ev)?;
Ok(ev)
}
fn tape_alias(&mut self, target: usize, span: Span) {
let Some(top) = self.recording.last_mut() else {
return;
};
let size = self
.anchors
.get(target)
.map_or_else(Size::default, |a| a.size);
top.2 = top.2.plus(size);
self.tape.push(Taped::Alias { target, span });
}
fn preload(&mut self, ev: Ev<'a>) -> Result<(), Error> {
if let Node::Alias { name } = ev.node {
let target = self.resolve_alias(name, ev.span)?;
self.tape_alias(target, ev.span);
return Ok(());
}
self.record(&ev);
Ok(())
}
fn record(&mut self, ev: &Ev<'a>) {
if let Some(name) = ev.anchor {
let index = self.anchors.len();
self.anchors.push(Anchor {
start: self.tape.len(),
end: None,
size: Size::default(),
});
self.names.insert(name, index);
self.recording
.push((index, self.src_depth, Size::default()));
}
match ev.node {
Node::SeqStart { .. } | Node::MapStart { .. } => {
self.src_depth = self.src_depth.saturating_add(1);
}
Node::SeqEnd | Node::MapEnd => self.src_depth = self.src_depth.saturating_sub(1),
_ => {}
}
if self.recording.is_empty() {
return;
}
self.tape.push(Taped::Ev(ev.clone()));
if let Some(top) = self.recording.last_mut() {
top.2 = top.2.plus(replay_cost(ev));
}
let end = self.tape.len();
let completes = matches!(ev.node, Node::Scalar { .. } | Node::SeqEnd | Node::MapEnd);
if !completes {
return;
}
while let Some(&(index, depth, size)) = self.recording.last() {
if depth != self.src_depth {
break;
}
if let Some(a) = self.anchors.get_mut(index) {
a.end = Some(end);
a.size = size;
}
self.recording.pop();
if let Some(outer) = self.recording.last_mut() {
outer.2 = outer.2.plus(size);
}
}
}
fn resolve_alias(&self, name: &str, span: Span) -> Result<usize, Error> {
let Some(&index) = self.names.get(name) else {
return Err(undefined_alias(name, span));
};
match self.anchors.get(index) {
Some(Anchor { end: Some(_), .. }) => Ok(index),
_ => Err(recursive_alias(name, span)),
}
}
fn push_replay(
&mut self,
target: usize,
alias: Span,
alias_replayed: bool,
) -> Result<(), Error> {
match self.anchors.get(target) {
Some(&Anchor {
start,
end: Some(end),
size,
}) => {
self.check_nodes(self.expanded.saturating_add(size.nodes), alias)?;
self.shared.meter.check(size.bytes, Some(alias))?;
let delivered = start >= self.preloaded
|| (alias_replayed && self.replay.last().is_some_and(|c| c.delivered));
self.replay.push(Cursor {
start,
pos: start,
end,
alias,
alias_replayed,
delivered,
});
Ok(())
}
_ => Err(Error::internal(
"alias to an unfinished anchor",
Some(alias),
)),
}
}
fn count_expanded(&mut self, span: Span) -> Result<(), Error> {
self.expanded = self.expanded.saturating_add(1);
self.check_nodes(self.expanded, span)
}
fn check_nodes(&mut self, total: usize, span: Span) -> Result<(), Error> {
let Some(budget) = self.limits.get_alias_budget(self.own_nodes) else {
return Ok(());
};
if total <= budget {
return Ok(());
}
let flat = self.limits.get_alias_budget(0) == self.limits.get_alias_budget(usize::MAX);
if !flat && self.nodes_before.is_none() {
self.nodes_before = Some(self.src.nodes_before().unwrap_or(0));
}
let own = self
.own_nodes
.saturating_add(self.nodes_before.unwrap_or(0));
match self.limits.get_alias_budget(own) {
Some(budget) if total > budget => Err(alias_budget_exceeded(budget, own, Some(span))),
_ => Ok(()),
}
}
fn track_depth(&mut self, ev: &Ev<'a>) -> Result<(), Error> {
match ev.node {
Node::SeqStart { .. } | Node::MapStart { .. } => {
self.depth = self.depth.saturating_add(1);
let limit = self.limits.get_max_depth();
if self.depth > limit {
return Err(Error::new(
code::DEPTH_LIMIT_EXCEEDED,
format!("nesting deeper than {limit} levels after alias expansion"),
Some(ev.span),
)
.with_help(format!("raise Limits::max_depth (currently {limit})")));
}
}
Node::SeqEnd | Node::MapEnd => self.depth = self.depth.saturating_sub(1),
_ => {}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
use super::*;
use crate::{Document, Stream, Value};
use oneyaml_core::Settings;
fn replayed<R>(f: impl FnOnce() -> R) -> (R, usize) {
let before = REPLAYED.with(std::cell::Cell::get);
let r = f();
(
r,
REPLAYED.with(std::cell::Cell::get).saturating_sub(before),
)
}
fn chain() -> String {
let mut s = String::from("l0: &l0 [x, x, x, x, x, x, x, x, x, x]\n");
for i in 1usize..7 {
let refs = vec![format!("*l{}", i.saturating_sub(1)); 10].join(", ");
s.push_str(&format!("l{i}: &l{i} [{refs}]\n"));
}
s
}
fn untrusted() -> Settings {
Settings::new().limits(Limits::untrusted())
}
#[test]
fn anchor_sizes_count_nested_aliases_and_saturate() {
let src = "a: &a [x, !t yy]\nb: &b [*a, *a]\nc: [*b]\n";
let feed = super::super::feed::Feed::new(
oneyaml_syntax::events_with(src, &Settings::new()),
src.len(),
);
let mut e = Expander::new(feed, &Limits::default());
let mut n = 0usize;
loop {
let ev = e.next().unwrap();
if ev.node == Node::StreamEnd {
break;
}
n = n.saturating_add(1);
}
assert!(n > 0);
let a = e.anchors[0].size;
assert_eq!(
a,
Size {
nodes: 3,
bytes: 1 + 2 + 2
}
);
assert_eq!(
e.anchors[1].size,
Size {
nodes: 7,
bytes: 10
}
);
let mut s = String::from("a0: &a0 [x]\n");
for i in 1usize..80 {
s.push_str(&format!(
"a{i}: &a{i} [*a{}, *a{}]\n",
i.saturating_sub(1),
i.saturating_sub(1)
));
}
let (r, n) = replayed(|| crate::from_str::<Value>(&s));
assert_eq!(r.unwrap_err().code(), code::ALIAS_BUDGET_EXCEEDED);
assert!(n < 1 << 21, "{n}");
}
#[test]
fn refusals_do_not_replay() {
let mut src = chain();
for i in 0..300 {
src.push_str(&format!("n{i}: [*l6]\n"));
}
for (s, budget) in [(Settings::new(), 1_000_000usize), (untrusted(), 100_000)] {
let (r, n) = replayed(|| crate::from_str_with::<Value>(&src, &s));
assert_eq!(r.unwrap_err().code(), code::ALIAS_BUDGET_EXCEEDED);
assert!(n <= budget.saturating_mul(2), "{n}");
let doc = Document::parse_with(src.as_str(), &s).unwrap();
let (refused, n) = replayed(|| {
(0..300)
.filter(|i| {
let node = doc.get(format!("n{i}")).unwrap();
node.deserialize::<Value>().unwrap_err().code()
== code::ALIAS_BUDGET_EXCEEDED
})
.count()
});
assert_eq!((refused, n), (300, 0));
}
let mut src = format!("a: &a {}\nb: [", "x".repeat(1 << 16));
src.push_str(&vec!["*a"; 1000].join(", "));
src.push_str("]\n");
let s = Settings::new().limits(Limits::default().alias_byte_budget(1 << 20));
let (r, n) = replayed(|| crate::from_str_with::<Value>(&src, &s));
assert_eq!(r.unwrap_err().code(), code::ALIAS_BYTE_BUDGET_EXCEEDED);
assert_eq!(n, 16);
let doc = Document::parse_with(src.as_str(), &s).unwrap();
let (r, n) = replayed(|| doc.get("b").unwrap().deserialize::<Value>());
assert_eq!(r.unwrap_err().code(), code::ALIAS_BYTE_BUDGET_EXCEEDED);
assert_eq!(n, 16);
let src = format!("a: &a {}\nb: [*a]\n", "x".repeat(1 << 16));
let s = Settings::new().limits(Limits::default().alias_byte_budget(1 << 15));
let doc = Document::parse_with(src.as_str(), &s).unwrap();
let (r, n) = replayed(|| {
(0..300)
.filter(|_| doc.get("b").unwrap().deserialize::<Value>().is_err())
.count()
});
assert_eq!((r, n), (300, 0));
}
#[test]
fn a_thousand_refused_documents_stay_within_their_budgets() {
let one = format!("{}n: [*l6]\n", chain());
let stream_src = vec![one.as_str(); 1000].join("---\n");
let budget = 1_000;
let s = Settings::new().limits(Limits::untrusted().alias_budget(budget));
let stream = Stream::parse_with(stream_src.as_str(), &s).unwrap();
assert_eq!(stream.len(), 1000);
let (refused, n) = replayed(|| {
stream
.documents()
.filter(|d| d.to_value().unwrap_err().code() == code::ALIAS_BUDGET_EXCEEDED)
.count()
});
assert_eq!(refused, 1000);
assert!(n <= budget.saturating_mul(2000), "{n}");
let docs: Vec<_> = crate::Deserializer::from_str_with(&stream_src, &s)
.map(<Value as serde::Deserialize>::deserialize)
.collect();
assert_eq!(docs.len(), 1);
assert_eq!(
docs[0].as_ref().unwrap_err().code(),
code::ALIAS_BUDGET_EXCEEDED
);
}
}