use crate::jqlang::{JqVal, NumLit};
use crate::ynode::{NodeMeta, Style};
use saphyr_parser::{Event, Parser, ScalarStyle, Span};
use std::collections::HashMap;
use std::rc::Rc;
thread_local! {
static INPUT_NAME: std::cell::RefCell<Rc<str>> = std::cell::RefCell::new(Rc::from("-"));
}
pub fn set_input_name(name: &str) {
INPUT_NAME.with(|n| *n.borrow_mut() = Rc::from(name));
}
pub fn documents(src: &str) -> Vec<JqVal> {
let name = INPUT_NAME.with(|n| n.borrow().clone());
documents_from(src, &name, 0)
}
pub fn documents_from(src: &str, file: &str, idx: u32) -> Vec<JqVal> {
let mut evs = Vec::new();
for item in Parser::new_from_str(src).keep_tags(true) {
match item {
Ok(pair) => evs.push(pair),
Err(_) => break,
}
}
let bytes = ByteMap::new(src);
let lines = LineIndex::new(src);
let comments = CommentMap::build(src, &evs, &bytes);
Composer {
evs: &evs,
i: 0,
src,
bytes,
lines,
comments,
anchors: HashMap::new(),
doc: 0,
file: Rc::from(file),
file_index: idx,
last_line: 0,
pending_head: None,
line_base: 0,
}
.run()
}
#[derive(Clone, Default)]
struct LineInfo {
text: Option<String>,
own: bool,
col: usize,
blank: bool,
claimed: bool,
}
struct CommentMap {
lines: Vec<LineInfo>,
}
impl CommentMap {
fn build(src: &str, evs: &[(Event<'_>, Span)], bytes: &ByteMap) -> CommentMap {
let mut masked: Vec<(usize, usize)> = evs
.iter()
.filter(|(e, _)| matches!(e, Event::Scalar(..)))
.map(|(_, sp)| (bytes.at(sp.start.index()), bytes.at(sp.end.index())))
.filter(|(a, b)| b > a)
.collect();
masked.sort_unstable();
let mut lines = vec![LineInfo::default()];
let mut at = 0usize;
for raw in src.split('\n') {
let mut info = LineInfo {
blank: raw.trim().is_empty(),
..LineInfo::default()
};
if let Some((col, text)) = find_comment(raw, at, &masked) {
info.own = raw[..col].trim().is_empty();
info.col = raw[..col].chars().count();
info.text = Some(text);
}
lines.push(info);
at += raw.len() + 1;
}
CommentMap { lines }
}
fn take_head(&mut self, line: usize) -> String {
let mut top = line;
while top > 1 {
match self.lines.get(top - 1) {
Some(l) if l.own && l.text.is_some() && !l.claimed => top -= 1,
_ => break,
}
}
self.take_block(top, line)
}
fn take_foot(&mut self, line: usize, col: usize) -> String {
let mut end = line + 1;
while let Some(l) = self.lines.get(end) {
if l.own && l.text.is_some() && !l.claimed && l.col == col {
end += 1;
} else {
break;
}
}
if end == line + 1 {
return String::new();
}
let terminated = match self.lines.get(end) {
None => true,
Some(l) => l.blank,
};
if !terminated {
return String::new();
}
self.take_block(line + 1, end)
}
fn take_block(&mut self, from: usize, to: usize) -> String {
let mut parts = Vec::new();
for l in from..to {
if let Some(info) = self.lines.get_mut(l) {
if let Some(t) = info.text.clone() {
info.claimed = true;
parts.push(t);
}
}
}
parts.join("\n")
}
fn take_line(&mut self, line: usize) -> String {
match self.lines.get_mut(line) {
Some(info) if !info.own && !info.claimed => match info.text.clone() {
Some(t) => {
info.claimed = true;
t
}
None => String::new(),
},
_ => String::new(),
}
}
}
fn find_comment(raw: &str, base: usize, masked: &[(usize, usize)]) -> Option<(usize, String)> {
let bytes = raw.as_bytes();
for (i, &b) in bytes.iter().enumerate() {
if b != b'#' {
continue;
}
if i > 0 && !matches!(bytes[i - 1], b' ' | b'\t') {
continue;
}
let abs = base + i;
if masked.iter().any(|&(s, e)| abs >= s && abs < e) {
continue;
}
let text = raw[i + 1..].strip_prefix(' ').unwrap_or(&raw[i + 1..]);
return Some((i, text.trim_end_matches('\r').to_string()));
}
None
}
struct ByteMap(Vec<usize>);
struct LineIndex(Vec<usize>);
impl LineIndex {
fn new(src: &str) -> LineIndex {
let mut v = vec![0usize];
for (i, b) in src.bytes().enumerate() {
if b == b'\n' {
v.push(i + 1);
}
}
LineIndex(v)
}
fn at(&self, byte: usize) -> (usize, usize) {
let line = self.0.partition_point(|&s| s <= byte).max(1);
(line, byte - self.0[line - 1])
}
}
impl ByteMap {
fn new(src: &str) -> ByteMap {
let mut v: Vec<usize> = src.char_indices().map(|(b, _)| b).collect();
v.push(src.len());
ByteMap(v)
}
fn at(&self, i: usize) -> usize {
*self.0.get(i).unwrap_or(self.0.last().unwrap_or(&0))
}
}
struct Composer<'a, 'input> {
evs: &'a [(Event<'input>, Span)],
i: usize,
src: &'a str,
bytes: ByteMap,
lines: LineIndex,
comments: CommentMap,
anchors: HashMap<usize, JqVal>,
doc: u32,
file: Rc<str>,
file_index: u32,
last_line: usize,
pending_head: Option<String>,
line_base: u32,
}
impl Composer<'_, '_> {
fn run(mut self) -> Vec<JqVal> {
let mut docs = Vec::new();
if let Some((_, sp)) = self
.evs
.iter()
.find(|(e, _)| !matches!(e, Event::StreamStart))
{
self.line_base = sp.start.line().saturating_sub(1) as u32;
}
while let Some((ev, _)) = self.evs.get(self.i) {
match ev {
Event::StreamEnd => break,
Event::DocumentStart(explicit) => {
let explicit = *explicit;
self.i += 1;
self.anchors.clear();
let first = self.evs.get(self.i).map_or(1, |(_, sp)| sp.start.line());
let head = self.comments.take_head(first);
let v = self.node(&[], None, false);
if !matches!(v.bare(), JqVal::Null) {
let foot = self.comments.take_foot(self.last_line, 0);
let v = if head.is_empty() && foot.is_empty() && !explicit {
v
} else {
v.with_meta(|m| {
if !head.is_empty() {
m.head = Rc::from(head.as_str());
}
if !foot.is_empty() {
m.foot = Rc::from(foot.as_str());
}
m.explicit_doc = explicit;
})
};
docs.push(v);
}
self.doc += 1;
}
_ => self.i += 1,
}
}
docs
}
fn marker_comment(&mut self, line: usize) -> String {
if line < 2 {
return String::new();
}
let prev = line - 1;
let start = self
.lines
.0
.get(prev.saturating_sub(1))
.copied()
.unwrap_or(0);
let end = self.src[start..]
.find('\n')
.map_or(self.src.len(), |n| start + n);
let text = &self.src[start..end];
match text.find('#') {
Some(at) if text[..at].trim() == "-" => self.comments.take_line(prev),
_ => String::new(),
}
}
fn indicator_col(&self, line: usize) -> u32 {
let start = self
.lines
.0
.get(line.saturating_sub(1))
.copied()
.unwrap_or(0);
let end = self.src[start..]
.find('\n')
.map_or(self.src.len(), |n| start + n);
match self.src[start..end].find(['|', '>']) {
Some(at) => self.src[start..start + at].chars().count() as u32 + 1,
None => 1,
}
}
fn span_text(&self, sp: &Span) -> &str {
let (a, b) = (
self.bytes.at(sp.start.index()),
self.bytes.at(sp.end.index()),
);
self.src.get(a..b).unwrap_or("")
}
fn line_of(&self, sp: &Span) -> u32 {
(sp.start.line() as u32)
.saturating_sub(self.line_base)
.max(1)
}
fn node(&mut self, path: &[JqVal], key: Option<Rc<JqVal>>, as_key: bool) -> JqVal {
let Some((ev, sp)) = self.evs.get(self.i) else {
return JqVal::Null;
};
let (ev, sp) = (ev.clone(), *sp);
self.i += 1;
let mut meta = NodeMeta {
line_no: self.line_of(&sp),
col_no: sp.start.col() as u32 + 1,
path: Rc::new(path.to_vec()),
key,
doc: self.doc,
file: self.file.clone(),
file_index: self.file_index,
..NodeMeta::default()
};
match ev {
Event::Scalar(text, style, aid, tag) => {
let tref = tag.as_ref().map(|t| t.as_ref());
let v = scalar(&text, style, tref);
meta.style = scalar_style(style);
meta.tag = tag_name(tref);
meta.anchor = self.anchor_of(aid, self.bytes.at(sp.start.index()));
self.anchor_pos(aid, self.bytes.at(sp.start.index()), &mut meta);
self.anchor(aid, &v);
let line = if style == ScalarStyle::Literal || style == ScalarStyle::Folded {
let l = sp.start.line().saturating_sub(1);
meta.line_no = (l as u32).saturating_sub(self.line_base).max(1);
meta.col_no = self.indicator_col(l);
l
} else {
sp.start.line()
};
self.last_line = self.last_line.max(sp.end.line());
if !as_key {
meta.line = Rc::from(self.comments.take_line(line).as_str());
}
let src_text = self.span_text(&sp);
meta.blank = style == ScalarStyle::Plain && src_text.is_empty();
meta.raw = raw_if_needed(src_text, style, &v);
JqVal::wrap(v, meta)
}
Event::Alias(aid) => {
let v = self.anchors.get(&aid).cloned().unwrap_or(JqVal::Null);
meta.alias = Rc::from(self.span_text(&sp).trim_start_matches('*'));
self.last_line = self.last_line.max(sp.end.line());
if !as_key {
meta.line = Rc::from(self.comments.take_line(sp.start.line()).as_str());
}
JqVal::wrap(v.bare().clone(), meta)
}
Event::SequenceStart(aid, tag) => {
let flow = self.src.as_bytes().get(self.bytes.at(sp.start.index())) == Some(&b'[');
meta.style = if flow { Style::Flow } else { Style::Block };
meta.tag = tag_name(tag.as_ref().map(|t| t.as_ref()));
meta.anchor = self.anchor_of(aid, self.bytes.at(sp.start.index()));
let mut items = Vec::new();
let icol = sp.start.col();
while !matches!(
self.evs.get(self.i).map(|(e, _)| e),
Some(Event::SequenceEnd) | None
) {
let mut p = path.to_vec();
p.push(JqVal::num(items.len() as f64));
let start = self.evs[self.i].1.start.line();
let head = self.comments.take_head(start);
let marker = self.marker_comment(start);
self.pending_head = (!marker.is_empty()).then(|| marker.clone());
let item = self.node(&p, None, false);
self.pending_head = None;
let foot = self.comments.take_foot(self.last_line, icol);
items.push(if head.is_empty() && foot.is_empty() && marker.is_empty() {
item
} else {
item.with_meta(|m| {
if !head.is_empty() {
m.head = Rc::from(head.as_str());
}
if !foot.is_empty() {
m.foot = Rc::from(foot.as_str());
}
m.marker = !marker.is_empty();
})
});
}
self.i += 1; let v = JqVal::arr(items);
self.anchor_pos(aid, self.bytes.at(sp.start.index()), &mut meta);
self.anchor(aid, &v);
JqVal::wrap(v, meta)
}
Event::MappingStart(aid, tag) => {
let flow = self.src.as_bytes().get(self.bytes.at(sp.start.index())) == Some(&b'{');
meta.style = if flow { Style::Flow } else { Style::Block };
meta.tag = tag_name(tag.as_ref().map(|t| t.as_ref()));
meta.anchor = self.anchor_of(aid, self.bytes.at(sp.start.index()));
let (v, written) = self.mapping(path);
meta.written = written;
self.anchor_pos(aid, self.bytes.at(sp.start.index()), &mut meta);
self.anchor(aid, &v);
JqVal::wrap(v, meta)
}
Event::DocumentEnd => {
self.i -= 1;
JqVal::Null
}
_ => JqVal::Null,
}
}
fn mapping(&mut self, path: &[JqVal]) -> (JqVal, Option<crate::ynode::Entries>) {
let mut pairs: Vec<(Rc<str>, JqVal)> = Vec::new();
let mut merges: Vec<(usize, JqVal)> = Vec::new();
let mut keys: Vec<JqVal> = Vec::new();
while !matches!(
self.evs.get(self.i).map(|(e, _)| e),
Some(Event::MappingEnd) | None
) {
let kline = self.evs[self.i].1.start.line();
let kcol = self.evs[self.i].1.start.col();
let k = self.node(&[], None, true);
let head = match self.pending_head.take() {
Some(h) => h,
None => self.comments.take_head(kline),
};
let k = if head.is_empty() {
k
} else {
k.with_meta(|m| {
m.head = Rc::from(head.as_str());
m.is_key = true;
})
};
let k = if k.meta().is_some_and(|m| m.is_key) {
k
} else {
k.with_meta(|m| m.is_key = true)
};
let key: Rc<str> = match k.bare() {
JqVal::Str(s) => s.clone(),
other => Rc::from(crate::jqlang::render_raw(other).as_str()),
};
let mut p = path.to_vec();
p.push(JqVal::Str(key.clone()));
let v = self.node(&p, Some(Rc::new(k.clone())), false);
let foot = self.comments.take_foot(self.last_line, kcol);
let k = if foot.is_empty() {
k
} else {
k.with_meta(|m| {
m.foot = Rc::from(foot.as_str());
m.is_key = true;
})
};
if &*key == "<<" {
merges.push((pairs.len(), v));
continue;
}
match pairs.iter().position(|(k2, _)| *k2 == key) {
Some(at) => {
pairs[at].1 = v;
keys[at] = k;
}
None => {
pairs.push((key, v));
keys.push(k);
}
}
}
self.i += 1; for (slot, k) in pairs.iter_mut().zip(keys.iter()) {
slot.1 = slot.1.with_meta(|m| m.key = Some(Rc::new(k.clone())));
}
if merges.is_empty() {
return (JqVal::obj(pairs), None);
}
let written = pairs.clone();
let mut shift = 0;
for (at, src) in merges.clone() {
let sources = match src.bare() {
JqVal::Arr(a) => a.to_vec(),
other => vec![other.clone()],
};
let mut insert: Vec<(Rc<str>, JqVal)> = Vec::new();
for s in sources {
if let JqVal::Obj(m) = s.bare() {
for (k, v) in m.iter() {
let taken = pairs.iter().chain(insert.iter()).any(|(k2, _)| k2 == k);
if !taken {
insert.push((k.clone(), v.clone()));
}
}
}
}
let n = insert.len();
pairs.splice(at + shift..at + shift, insert);
shift += n;
}
let mut written = written;
for (shift, (at, src)) in merges.into_iter().enumerate() {
written.insert(at + shift, (Rc::from("<<"), src));
}
(JqVal::obj(pairs), Some(Rc::new(written)))
}
fn anchor(&mut self, aid: usize, v: &JqVal) {
if aid > 0 {
self.anchors.insert(aid, v.clone());
}
}
fn anchor_of(&self, aid: usize, at: usize) -> Rc<str> {
if aid == 0 {
return Rc::from("");
}
Rc::from(self.anchor_name(at).0.as_str())
}
fn anchor_pos(&self, aid: usize, at: usize, meta: &mut NodeMeta) {
if aid == 0 {
return;
}
let (_, off) = self.anchor_name(at);
let Some(off) = off else { return };
let (line, col) = self.lines.at(off);
meta.line_no = (line as u32).saturating_sub(self.line_base).max(1);
meta.col_no = col as u32 + 1;
}
fn anchor_name(&self, at: usize) -> (String, Option<usize>) {
let b = self.src.as_bytes();
let mut i = at;
for _ in 0..2 {
while i > 0 && matches!(b[i - 1], b' ' | b'\t' | b'\n' | b'\r') {
i -= 1;
}
let end = i;
while i > 0 && !matches!(b[i - 1], b' ' | b'\t' | b'\n' | b'\r' | b':' | b'-') {
i -= 1;
}
match b.get(i) {
Some(b'&') => {
return (
String::from_utf8_lossy(&b[i + 1..end]).into_owned(),
Some(i),
);
}
Some(b'!') => continue,
_ => return (String::new(), None),
}
}
(String::new(), None)
}
}
fn raw_if_needed(text: &str, style: ScalarStyle, v: &JqVal) -> Rc<str> {
if style == ScalarStyle::Folded {
return Rc::from(text);
}
if style != ScalarStyle::Plain || text.is_empty() {
return crate::ynode::empty_str();
}
match v {
JqVal::Num(_, NumLit::Text(lit)) => {
return if &**lit == text {
crate::ynode::empty_str()
} else {
Rc::from(text)
}
}
JqVal::Str(s) if &**s == text && !crate::ynode::needs_quoting(text) => {
return crate::ynode::empty_str()
}
_ => {}
}
let written = crate::ynode::quote_scalar_value(v);
if written == text {
crate::ynode::empty_str()
} else {
Rc::from(text)
}
}
fn tag_name(tag: Option<&saphyr_parser::Tag>) -> Rc<str> {
match tag {
None => Rc::from(""),
Some(t) if t.handle == "tag:yaml.org,2002:" => Rc::from(format!("!!{}", t.suffix).as_str()),
Some(t) if t.handle == "!" => Rc::from(format!("!{}", t.suffix).as_str()),
Some(t) => Rc::from(format!("{}{}", t.handle, t.suffix).as_str()),
}
}
fn scalar_style(style: ScalarStyle) -> Style {
match style {
ScalarStyle::Plain => Style::Plain,
ScalarStyle::SingleQuoted => Style::Single,
ScalarStyle::DoubleQuoted => Style::Double,
ScalarStyle::Literal => Style::Literal,
ScalarStyle::Folded => Style::Folded,
}
}
fn scalar(text: &str, style: ScalarStyle, tag: Option<&saphyr_parser::Tag>) -> JqVal {
if let Some(t) = tag {
if t.handle == "tag:yaml.org,2002:" {
match t.suffix.as_ref() {
"str" => return JqVal::str(text),
"null" => return JqVal::Null,
"bool" => return parse_bool(text).map_or(JqVal::Null, JqVal::Bool),
"int" | "float" => {
return parse_int(text)
.or_else(|| parse_float(text))
.unwrap_or(JqVal::Null)
}
_ => {}
}
}
}
if style != ScalarStyle::Plain {
return JqVal::str(text);
}
plain(text)
}
fn plain(text: &str) -> JqVal {
if text.is_empty() || matches!(text, "null" | "Null" | "NULL" | "~") {
return JqVal::Null;
}
if let Some(b) = parse_bool(text) {
return JqVal::Bool(b);
}
parse_int(text)
.or_else(|| parse_float(text))
.unwrap_or_else(|| JqVal::str(text))
}
fn parse_bool(text: &str) -> Option<bool> {
match text {
"true" | "True" | "TRUE" => Some(true),
"false" | "False" | "FALSE" => Some(false),
_ => None,
}
}
fn parse_int(text: &str) -> Option<JqVal> {
let (neg, body) = match text.strip_prefix('-') {
Some(rest) => (true, rest),
None => (false, text.strip_prefix('+').unwrap_or(text)),
};
if body.starts_with(['+', '-']) {
return None;
}
let (radix, digits) = match body {
b if b.len() > 2 && (b.starts_with("0x") || b.starts_with("0X")) => (16, &b[2..]),
b if b.len() > 2 && (b.starts_with("0o") || b.starts_with("0O")) => (8, &b[2..]),
b => (10, b),
};
let clean: String = digits.chars().filter(|c| *c != '_').collect();
if clean.is_empty() || digits.starts_with('_') || !clean.chars().all(|c| c.is_digit(radix)) {
return None;
}
let n = match i64::from_str_radix(&clean, radix) {
Ok(v) => v as f64,
Err(_) if radix == 10 => clean.parse::<f64>().ok()?,
Err(_) => u64::from_str_radix(&clean, radix).ok()? as f64,
};
let n = if neg { -n } else { n };
if radix == 10 && clean.len() == digits.len() {
return Some(crate::jqlang::num_from_literal(
n,
text.trim_start_matches('+'),
));
}
Some(JqVal::num(n))
}
fn parse_float(text: &str) -> Option<JqVal> {
let body = text.strip_prefix('+').unwrap_or(text);
if body.starts_with(['+', '-']) && !body.starts_with('-') {
return None;
}
match body {
".inf" | ".Inf" | ".INF" => return Some(JqVal::num(f64::INFINITY)),
"-.inf" | "-.Inf" | "-.INF" => return Some(JqVal::num(f64::NEG_INFINITY)),
".nan" | ".NaN" | ".NAN" => return Some(JqVal::num(f64::NAN)),
_ => {}
}
let clean: String = body.chars().filter(|c| *c != '_').collect();
if clean
.chars()
.any(|c| !matches!(c, '0'..='9' | '.' | 'e' | 'E' | '+' | '-'))
{
return None;
}
let n = clean.parse::<f64>().ok()?;
if clean.len() != body.len() {
return Some(JqVal::num(n));
}
let lit = if body.ends_with('.') {
format!("{body}0")
} else {
body.to_string()
};
Some(crate::jqlang::num_from_literal(n, &lit))
}