use std::collections::{HashMap, HashSet};
use std::ffi::CStr;
use std::mem::MaybeUninit;
use std::rc::Rc;
use anyhow::{Context, bail, ensure};
use serde_yaml_ng::Value;
const UNSUPPORTED_TAG: &str = "a YAML tag in the config file is not supported";
const UNSUPPORTED_KEY: &str = "a tagged or aliased mapping key in the config file is not supported";
const COLLECTION_KEY: &str = "a mapping or sequence used as a mapping key in the config file is not supported";
const QUOTED_MERGE_KEY: &str = "a quoted << mapping key in the config file is not supported";
const REPEATED_ANCHOR: &str = "a YAML anchor name is defined more than once in the config file";
const PLUGIN_PIN: &str =
"a plugin's store.version or store.release-tag in the config file is a !!binary value, which is not supported";
const MAX_COPIES: usize = 1 << 20;
const MAX_VISITS: usize = 1 << 22;
pub(super) fn go_base64(text: &[u8]) -> Option<Vec<u8>> {
use base64::engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig};
use base64::{Engine as _, alphabet};
let engine = GeneralPurpose::new(
&alphabet::STANDARD,
GeneralPurposeConfig::new()
.with_decode_allow_trailing_bits(true)
.with_decode_padding_mode(DecodePaddingMode::RequireCanonical),
);
let payload: Vec<u8> = text.iter().copied().filter(|b| !matches!(b, b'\r' | b'\n')).collect();
engine.decode(payload).ok()
}
pub(super) fn decode(text: &str, value: &mut Value, literals: Option<&[&str]>, mode: Mode) -> anyhow::Result<()> {
if mode != Mode::Plugins
&& !text.contains(['!', '&'])
&& !needs_walk(value)
&& !literals.is_some_and(|at| scalar_literal_field(value, &mut at.iter().map(|k| Some(*k)).collect()))
{
return Ok(());
}
let mut walk = Walk {
events: events(text)?.into_iter(),
defined: HashSet::new(),
sequences: HashSet::new(),
anchors: HashMap::new(),
anchored: 0,
last_field: None,
literals: literals.map(|at| at.iter().map(|k| Some(Rc::from(*k))).collect()),
keys: Vec::new(),
found: Vec::new(),
copies: 0,
visits: 0,
mode,
};
if let Some(root) = walk.events.next() {
walk.node(root, &mut Vec::new(), false)?;
}
ensure!(
!walk.found.iter().any(|f| f.kind == Kind::Binary && plugin_pin(&f.keys)),
PLUGIN_PIN
);
let mut found: Vec<Found> = walk
.found
.into_iter()
.filter(|f| f.kind != Kind::Literal(false))
.collect();
found.sort_by(|a, b| a.path.cmp(&b.path));
apply(value, 0, &found)
}
fn needs_walk(value: &Value) -> bool {
match value {
Value::Mapping(m) => m.iter().any(|(k, v)| {
matches!(k, Value::Mapping(_) | Value::Sequence(_)) || k.as_str() == Some("<<") || needs_walk(v)
}),
Value::Sequence(items) => items.iter().any(needs_walk),
Value::Tagged(tagged) => needs_walk(&tagged.value),
_ => false,
}
}
fn apply(node: &mut Value, depth: usize, patches: &[Found]) -> anyhow::Result<()> {
let Some(first) = patches.first() else {
return Ok(());
};
if first.path.len() == depth {
*node = first.value.clone();
return Ok(());
}
let mut node = node;
while let Value::Tagged(tagged) = node {
node = &mut tagged.value;
}
let children: Box<dyn Iterator<Item = &mut Value>> = match node {
Value::Mapping(m) => Box::new(m.values_mut()),
Value::Sequence(items) => Box::new(items.iter_mut()),
_ => bail!("config YAML changed while it was read"),
};
let mut rest = patches;
for (i, child) in children.enumerate() {
if rest.is_empty() {
break;
}
let n = rest.iter().take_while(|f| f.path[depth] == i).count();
apply(child, depth + 1, &rest[..n])?;
rest = &rest[n..];
}
ensure!(rest.is_empty(), "config YAML changed while it was read");
Ok(())
}
fn scalar_literal_field<'a>(value: &'a Value, keys: &mut Vec<Option<&'a str>>) -> bool {
match value {
Value::Mapping(m) => m.iter().any(|(k, v)| {
keys.push(Some(k.as_str().unwrap_or("")));
let found = scalar_literal_field(v, keys);
keys.pop();
found
}),
Value::Sequence(items) => items.iter().any(|v| {
keys.push(None);
let found = scalar_literal_field(v, keys);
keys.pop();
found
}),
Value::Number(_) | Value::Bool(_) | Value::Tagged(_) => literal_field(keys),
_ => false,
}
}
pub(super) fn as_reread(value: &mut Value) {
fn walk(value: &mut Value, keys: &mut Vec<Option<String>>) {
match value {
Value::Mapping(m) => {
for (k, v) in m.iter_mut() {
keys.push(Some(k.as_str().unwrap_or("").to_owned()));
walk(v, keys);
keys.pop();
}
}
Value::Sequence(items) => {
for v in items {
keys.push(None);
walk(v, keys);
keys.pop();
}
}
Value::Number(_) | Value::Bool(_) if literal_field(keys) => *value = Value::String(super::go_string(value)),
_ => {}
}
}
walk(value, &mut Vec::new());
}
fn named<K: AsRef<str>>(keys: &[Option<K>]) -> Vec<Option<&str>> {
let mut named = Vec::with_capacity(keys.len());
let mut keys = keys.iter().map(|k| k.as_ref().map(AsRef::as_ref)).peekable();
while let Some(key) = keys.next() {
if key == Some("<<") {
keys.next_if(Option::is_none);
} else {
named.push(key);
}
}
named
}
fn literal_field<K: AsRef<str>>(keys: &[Option<K>]) -> bool {
super::schema::is_string(&super::document::v8_path(&named(keys)))
}
fn plugin_config<K: AsRef<str>>(keys: &[Option<K>]) -> bool {
matches!(named(keys).as_slice(), [Some("plugins"), Some("configs"), Some(_), ..])
}
fn plugin_pin<K: AsRef<str>>(keys: &[Option<K>]) -> bool {
matches!(
named(keys).as_slice(),
[
Some("plugins"),
Some("configs"),
Some(_),
Some("store"),
Some("version" | "release-tag")
]
)
}
enum Event {
Scalar {
anchor: Option<Vec<u8>>,
tag: Option<Vec<u8>>,
plain: bool,
value: Vec<u8>,
},
Alias(Vec<u8>),
Start {
anchor: Option<Vec<u8>>,
mapping: bool,
},
End,
}
fn events(text: &str) -> anyhow::Result<Vec<Event>> {
events_with(text, None)
}
pub(super) struct Plain {
pub span: std::ops::Range<usize>,
pub value: Vec<u8>,
pub key: bool,
pub timestamp_tag: Option<usize>,
}
pub(super) fn plain_scalars(text: &str) -> anyhow::Result<Vec<Plain>> {
let mut out = Vec::new();
events_with(text, Some(&mut out))?;
Ok(out)
}
fn events_with(text: &str, mut plain: Option<&mut Vec<Plain>>) -> anyhow::Result<Vec<Event>> {
let nul_escape = text.contains("%00");
let bom = if text.starts_with('\u{feff}') { 3 } else { 0 };
struct Parser(Box<MaybeUninit<unsafe_libyaml::yaml_parser_t>>);
impl Drop for Parser {
fn drop(&mut self) {
unsafe { unsafe_libyaml::yaml_parser_delete(self.0.as_mut_ptr()) }
}
}
fn bytes(ptr: *const u8) -> Option<Vec<u8>> {
(!ptr.is_null()).then(|| unsafe { CStr::from_ptr(ptr.cast()) }.to_bytes().to_vec())
}
let mut out = Vec::new();
let mut open: Vec<(bool, usize)> = Vec::new();
unsafe {
let mut raw = Box::new(MaybeUninit::<unsafe_libyaml::yaml_parser_t>::uninit());
ensure!(
!unsafe_libyaml::yaml_parser_initialize(raw.as_mut_ptr()).fail,
"failed to start the YAML parser"
);
let mut parser = Parser(raw);
let p = parser.0.as_mut_ptr();
unsafe_libyaml::yaml_parser_set_encoding(p, unsafe_libyaml::YAML_UTF8_ENCODING);
unsafe_libyaml::yaml_parser_set_input_string(p, text[bom..].as_ptr(), (text.len() - bom) as _);
loop {
let mut event = MaybeUninit::<unsafe_libyaml::yaml_event_t>::uninit();
ensure!(
!unsafe_libyaml::yaml_parser_parse(p, event.as_mut_ptr()).fail,
"failed to parse config file"
);
let event = event.assume_init_mut();
let kind = event.type_;
if nul_escape {
let tagged = match kind {
unsafe_libyaml::YAML_SCALAR_EVENT => !event.data.scalar.tag.is_null(),
unsafe_libyaml::YAML_SEQUENCE_START_EVENT => !event.data.sequence_start.tag.is_null(),
unsafe_libyaml::YAML_MAPPING_START_EVENT => !event.data.mapping_start.tag.is_null(),
unsafe_libyaml::YAML_DOCUMENT_START_EVENT => true,
_ => false,
};
let (start, end) = (
event.start_mark.index as usize + bom,
event.end_mark.index as usize + bom,
);
let span = text.get(start..end);
if tagged && span.is_none_or(|s| s.contains("%00")) {
unsafe_libyaml::yaml_event_delete(event);
bail!(UNSUPPORTED_TAG);
}
}
let key = match kind {
unsafe_libyaml::YAML_SCALAR_EVENT
| unsafe_libyaml::YAML_ALIAS_EVENT
| unsafe_libyaml::YAML_SEQUENCE_START_EVENT
| unsafe_libyaml::YAML_MAPPING_START_EVENT => open.last_mut().is_some_and(|(mapping, nodes)| {
*nodes += 1;
*mapping && *nodes % 2 == 1
}),
_ => false,
};
match kind {
unsafe_libyaml::YAML_SEQUENCE_START_EVENT => open.push((false, 0)),
unsafe_libyaml::YAML_MAPPING_START_EVENT => open.push((true, 0)),
unsafe_libyaml::YAML_SEQUENCE_END_EVENT | unsafe_libyaml::YAML_MAPPING_END_EVENT => {
open.pop();
}
_ => {}
}
let next = match kind {
unsafe_libyaml::YAML_SCALAR_EVENT => {
let s = &event.data.scalar;
let value = std::slice::from_raw_parts(s.value, s.length as usize);
let end = event.end_mark.index as usize + bom;
let timestamp = bytes(s.tag).is_some_and(|t| t == super::timestamp::TAG.as_bytes());
let folded_start = || {
let anchor = bytes(s.anchor).map_or(0, |a| 1 + a.len());
let after = event.start_mark.index as usize + bom + anchor;
let rest = text.get(after..end)?;
let mut comment = false;
let skipped = rest
.char_indices()
.find(|&(_, c)| {
let line_break = matches!(c, '\r' | '\n' | '\u{85}' | '\u{2028}' | '\u{2029}');
if line_break {
comment = false;
} else if c == '#' {
comment = true;
}
!(comment || line_break || matches!(c, ' ' | '\t'))
})
.map(|(i, _)| i)?;
let first = after + skipped;
(text.as_bytes().get(first) == value.first()).then_some(first)
};
let span = end
.checked_sub(value.len())
.filter(|&start| text.as_bytes().get(start..end) == Some(value))
.or_else(|| s.tag.is_null().then(folded_start).flatten())
.map(|start| start..end);
if let Some(plain) = plain.as_deref_mut()
&& s.style == unsafe_libyaml::YAML_PLAIN_SCALAR_STYLE
&& (s.tag.is_null() || timestamp)
&& let Some(span) = span
{
plain.push(Plain {
span,
value: value.to_vec(),
key,
timestamp_tag: timestamp.then_some(event.start_mark.index as usize + bom),
});
}
Some(Event::Scalar {
anchor: bytes(s.anchor),
tag: bytes(s.tag),
plain: s.style == unsafe_libyaml::YAML_PLAIN_SCALAR_STYLE,
value: std::slice::from_raw_parts(s.value, s.length as usize).to_vec(),
})
}
unsafe_libyaml::YAML_ALIAS_EVENT => bytes(event.data.alias.anchor).map(Event::Alias),
unsafe_libyaml::YAML_SEQUENCE_START_EVENT => Some(Event::Start {
anchor: bytes(event.data.sequence_start.anchor),
mapping: false,
}),
unsafe_libyaml::YAML_MAPPING_START_EVENT => Some(Event::Start {
anchor: bytes(event.data.mapping_start.anchor),
mapping: true,
}),
unsafe_libyaml::YAML_SEQUENCE_END_EVENT | unsafe_libyaml::YAML_MAPPING_END_EVENT => Some(Event::End),
_ => None,
};
unsafe_libyaml::yaml_event_delete(event);
out.extend(next);
if kind == unsafe_libyaml::YAML_STREAM_END_EVENT {
return Ok(out);
}
}
}
}
fn is_null(value: &[u8]) -> bool {
matches!(value, b"" | b"~" | b"null" | b"Null" | b"NULL")
}
fn resolves_to_text(value: &[u8]) -> bool {
const WORDS: &[&[u8]] = &[b"true", b"false", b"yes", b"no", b"on", b"off", b"y", b"n"];
value.first().is_some_and(u8::is_ascii_alphabetic) && !WORDS.iter().any(|w| value.eq_ignore_ascii_case(w))
}
fn rewrite(tag: &[u8], plain: bool, value: Vec<u8>) -> anyhow::Result<Option<Value>> {
let text = || {
String::from_utf8(value.clone())
.ok()
.context("config YAML is not UTF-8")
};
Ok(Some(match tag {
b"tag:yaml.org,2002:binary" | b"!!binary" => {
let bytes = go_base64(&value).context("yaml: !!binary value contains invalid base64 data")?;
let text = String::from_utf8(bytes)
.ok()
.context("a !!binary value in the config file is not UTF-8 text")?;
Value::String(text)
}
b"!" if plain && is_null(&value) => Value::Null,
b"!" if plain && !resolves_to_text(&value) => bail!(UNSUPPORTED_TAG),
b"!" | b"!!str" => Value::String(text()?),
b"!!null" if is_null(&value) => Value::Null,
t if t.starts_with(b"!!") => bail!(UNSUPPORTED_TAG),
t if t.starts_with(b"!") => Value::String(text()?),
_ => return Ok(None),
}))
}
#[derive(Clone)]
struct Found {
path: Vec<usize>,
keys: Vec<Key>,
value: Value,
kind: Kind,
}
type Key = Option<Rc<str>>;
#[derive(Clone, Copy, PartialEq)]
enum Kind {
Tag,
Binary,
Literal(bool),
}
struct Walk {
events: std::vec::IntoIter<Event>,
defined: HashSet<Vec<u8>>,
sequences: HashSet<Vec<u8>>,
anchors: HashMap<Vec<u8>, Vec<Found>>,
anchored: usize,
last_field: Option<(Vec<Key>, bool)>,
literals: Option<Vec<Key>>,
keys: Vec<Key>,
found: Vec<Found>,
copies: usize,
visits: usize,
mode: Mode,
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub(super) enum Mode {
File,
Body,
Plugins,
}
pub const PLAIN: &str = "!cpa-plain";
pub fn double_quoted(text: &str) -> String {
let mut out = String::with_capacity(text.len() + 2);
out.push('"');
for c in text.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\t' => out.push_str("\\t"),
'\r' => out.push_str("\\r"),
'\u{2028}' => out.push_str("\\L"),
'\u{2029}' => out.push_str("\\P"),
'\u{feff}' => out.push_str("\\uFEFF"),
'\u{fffe}' => out.push_str("\\uFFFE"),
'\u{ffff}' => out.push_str("\\uFFFF"),
c if c < ' ' || ('\u{7f}'..='\u{9f}').contains(&c) => out.push_str(&format!("\\x{:02X}", c as u32)),
c => out.push(c),
}
}
out.push('"');
out
}
impl Walk {
fn next(&mut self) -> anyhow::Result<Event> {
self.events.next().context("config YAML ended early")
}
fn define(&mut self, anchor: &Option<Vec<u8>>) -> anyhow::Result<()> {
if let Some(name) = anchor {
ensure!(self.defined.insert(name.clone()), REPEATED_ANCHOR);
}
Ok(())
}
fn text_value(&self, text: String, tag: Option<&[u8]>, plain: bool) -> Value {
if self.mode != Mode::Plugins {
return Value::String(text);
}
let text = match tag.and_then(|t| t.strip_prefix(b"tag:yaml.org,2002:")) {
Some(name) => {
let name = String::from_utf8_lossy(name);
let text = if plain { text } else { double_quoted(&text) };
format!("!!{name} {text}")
}
None => text,
};
Value::Tagged(Box::new(serde_yaml_ng::value::TaggedValue {
tag: serde_yaml_ng::value::Tag::new(PLAIN),
value: Value::String(text),
}))
}
fn literal(&mut self, keys: &[Key], anchored: bool) -> Option<Kind> {
let at = self.literals.as_ref()?;
let field = match &self.last_field {
Some((last, field)) if last.as_slice() == keys => *field,
_ => {
let field = if self.mode == Mode::Plugins {
plugin_config(keys)
} else if at.is_empty() {
literal_field(keys)
} else {
literal_field(&[at.as_slice(), keys].concat())
};
self.last_field = Some((keys.to_vec(), field));
field
}
};
(field || anchored).then_some(Kind::Literal(field))
}
fn node(&mut self, event: Event, path: &mut Vec<usize>, key: bool) -> anyhow::Result<()> {
let start = self.found.len();
let anchor = match event {
Event::Scalar {
anchor,
tag,
plain,
value,
} => {
self.define(&anchor)?;
let anchored = anchor.is_some() || self.anchored > 0;
let literal = match &tag {
Some(tag) => {
matches!(
tag.as_slice(),
b"tag:yaml.org,2002:int" | b"tag:yaml.org,2002:float" | b"tag:yaml.org,2002:bool"
) || (self.mode == Mode::Plugins && tag.as_slice() == b"tag:yaml.org,2002:timestamp")
}
None => plain && !is_null(&value) && !resolves_to_text(&value),
};
ensure!(tag.is_none() || !key, UNSUPPORTED_KEY);
if literal {
if key {
if let Some(name) = &anchor {
let text = String::from_utf8(value).ok().context("config YAML is not UTF-8")?;
let found = Found {
path: Vec::new(),
keys: Vec::new(),
value: self.text_value(text, None, true),
kind: Kind::Literal(false),
};
self.anchors.insert(name.clone(), vec![found]);
return Ok(());
}
} else if let Some(kind) = {
let keys = std::mem::take(&mut self.keys);
let kind = self.literal(&keys, anchored);
self.keys = keys;
kind
} {
let text = String::from_utf8(value).ok().context("config YAML is not UTF-8")?;
let value = self.text_value(text, tag.as_deref(), plain);
self.found.push(Found {
path: path.clone(),
keys: self.keys.clone(),
value,
kind,
});
}
} else if let Some(tag) = tag {
let kind = match tag.as_slice() {
b"tag:yaml.org,2002:binary" | b"!!binary" => Kind::Binary,
_ => Kind::Tag,
};
if let Some(v) = rewrite(&tag, plain, value)? {
self.found.push(Found {
path: path.clone(),
keys: self.keys.clone(),
value: v,
kind,
});
}
}
anchor
}
Event::Alias(name) => {
ensure!(!key, UNSUPPORTED_KEY);
self.visits += self.anchors.get(&name).map_or(0, Vec::len);
ensure!(self.visits <= MAX_VISITS, "config YAML aliases expand too far");
let copies = self.anchors.get(&name).cloned().unwrap_or_default();
for mut below in copies {
let keys = [self.keys.as_slice(), &below.keys].concat();
if let Kind::Literal(_) = below.kind {
match self.literal(&keys, self.anchored > 0) {
Some(kind) => below.kind = kind,
None => continue,
}
}
self.copies += 1;
ensure!(self.copies <= MAX_COPIES, "config YAML aliases expand too far");
self.found.push(Found {
path: [path.as_slice(), &below.path].concat(),
keys,
..below
});
}
None
}
Event::Start { anchor, mapping } => {
ensure!(!key, COLLECTION_KEY);
self.define(&anchor)?;
if let Some(name) = anchor.as_ref().filter(|_| !mapping) {
self.sequences.insert(name.clone());
}
if anchor.is_some() {
self.anchored += 1;
}
let mut i = 0;
loop {
let event = self.next()?;
if matches!(event, Event::End) {
break;
}
if mapping {
let merge = matches!(&event, Event::Scalar { tag: None, value, .. } if value == b"<<");
ensure!(
!merge || self.mode == Mode::Body || matches!(event, Event::Scalar { plain: true, .. }),
QUOTED_MERGE_KEY
);
let name: Rc<str> = match &event {
Event::Scalar { value, .. } => String::from_utf8_lossy(value).into(),
_ => "".into(),
};
self.node(event, path, true)?;
let value = self.next()?;
if merge && let Event::Alias(name) = &value {
ensure!(!self.sequences.contains(name), super::MERGE_NOT_MAPS);
}
path.push(i);
self.keys.push(Some(name));
self.node(value, path, false)?;
} else {
path.push(i);
self.keys.push(None);
self.node(event, path, false)?;
}
path.pop();
self.keys.pop();
i += 1;
}
if anchor.is_some() {
self.anchored -= 1;
}
anchor
}
Event::End => bail!("config YAML is malformed"),
};
if let Some(name) = anchor {
let below = self.found[start..]
.iter()
.map(|f| Found {
path: f.path[path.len()..].to_vec(),
keys: f.keys[path.len()..].to_vec(),
..f.clone()
})
.collect();
self.anchors.insert(name, below);
}
Ok(())
}
}