use crate::block::parse_block_range_checked;
use crate::block_scalar::build_block_scalar;
use crate::chars::is_trim_whitespace;
use crate::errors::{Diagnostic, LimaDiagnosticCode as Code, LimaError};
use crate::flow::parse_flow_or_scalar_value_checked;
use crate::normalize::{
begin_warning_collection, check_duplicate_key, check_key_length, finish_warning_collection,
DOCUMENT_SIZE_LIMIT, NESTING_DEPTH_LIMIT, TOP_LEVEL_KEY_LIMIT,
};
use crate::scalars::{
is_valid_key, physical_raw, separator_index, strip_comment, strip_key_quotes,
};
use crate::structure::{check_range_lines, RangeOwner};
use crate::value::{
Builder, LimaValue, PlainBuilder, PositionedBuilder, PositionedValue, ReferenceSource,
};
struct TopKey {
line: u32,
key_start: usize,
value_start: usize,
is_block: bool,
key: String,
}
fn find_key_sep(s: &str) -> Option<usize> {
let b = s.as_bytes();
let first = *b.first()?;
let separated = |i: usize| matches!(b.get(i), Some(b' ' | b'\t'));
if first == b'\'' {
let mut i = 1;
while i < b.len() && b[i] != first {
i += 1;
}
if b.get(i + 1) == Some(&b':') && separated(i + 2) {
return Some(i + 1);
}
return None;
}
if first == b'"' {
let mut i = 1;
while i < b.len() && b[i] != b'"' {
i += if b[i] == b'\\' && i + 1 < b.len() {
2
} else {
1
};
}
if b.get(i + 1) == Some(&b':') && separated(i + 2) {
return Some(i + 1);
}
return None;
}
separator_index(s)
}
fn scan_top_keys(source: &str, strict: bool) -> Result<Vec<TopKey>, LimaError> {
let mut result = Vec::new();
let bytes = source.as_bytes();
let mut pos = 0usize;
let mut line_no = 1u32;
while pos <= bytes.len() {
let line_end = source[pos..]
.find('\n')
.map(|p| pos + p)
.unwrap_or(bytes.len());
let line = &source[pos..line_end];
if !line.is_empty() && !line.starts_with([' ', '\t', '#']) {
if let Some(colon_pos) = find_key_sep(line) {
let key_raw = &line[..colon_pos];
if is_valid_key(key_raw) {
let key = strip_key_quotes(key_raw, strict, line_no)?;
result.push(TopKey {
line: line_no,
key_start: pos,
value_start: pos + colon_pos + 2,
is_block: false,
key,
});
}
} else if let Some(key_part) = line.strip_suffix(':') {
if !key_part.is_empty() && is_valid_key(key_part) {
let key = strip_key_quotes(key_part, strict, line_no)?;
let value_start = if line_end < bytes.len() {
line_end + 1
} else {
line_end
};
result.push(TopKey {
line: line_no,
key_start: pos,
value_start,
is_block: true,
key,
});
}
}
}
if line_end >= bytes.len() {
break;
}
pos = line_end + 1;
line_no += 1;
}
Ok(result)
}
fn has_space_before_colon(line: &str) -> bool {
let b = line.as_bytes();
let Some("e) = b.first() else {
return false;
};
if quote != b'\'' && quote != b'"' {
return false;
}
let mut i = 1;
if quote == b'\'' {
while i < b.len() && b[i] != b'\'' {
i += 1;
}
} else {
while i < b.len() && b[i] != b'"' {
i += if b[i] == b'\\' && i + 1 < b.len() {
2
} else {
1
};
}
}
if i >= b.len() {
return false;
} i += 1; let ws_start = i;
while i < b.len() && (b[i] == b' ' || b[i] == b'\t') {
i += 1;
}
i > ws_start && b.get(i) == Some(&b':')
}
fn inline_value_start(source: &str, start: usize, end: usize) -> usize {
source[start..end]
.char_indices()
.find(|&(_, c)| c == '\n' || !is_trim_whitespace(c))
.map_or(end, |(i, _)| start + i)
}
fn bare_key_block_start(source: &str, value_start: usize, end: usize) -> Option<usize> {
let rest = &source[value_start..end];
if !(rest.is_empty() || rest.starts_with(['\n', '#'])) {
return None;
}
let line_end = rest.find('\n').map_or(end, |p| value_start + p);
Some((line_end + 1).min(end))
}
fn line_at(source: &str, byte_pos: usize) -> u32 {
1 + source[..byte_pos].bytes().filter(|&c| c == b'\n').count() as u32
}
thread_local! {
static TAB_ADJUST: std::cell::RefCell<Option<Vec<usize>>> = const { std::cell::RefCell::new(None) };
}
pub(crate) fn tab_adjust_slice(from_line: u32, count: usize) -> Option<Vec<usize>> {
TAB_ADJUST.with(|t| {
t.borrow().as_ref().map(|v| {
let start = from_line.saturating_sub(1) as usize;
(0..count)
.map(|i| v.get(start + i).copied().unwrap_or(0))
.collect()
})
})
}
fn expand_leading_tabs(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut adjust: Vec<usize> = Vec::new();
for (i, line) in s.split('\n').enumerate() {
if i > 0 {
out.push('\n');
}
let leading_end = line
.find(|c: char| c != ' ' && c != '\t')
.unwrap_or(line.len());
let (leading, rest) = line.split_at(leading_end);
let tabs = leading.chars().filter(|&c| c == '\t').count();
adjust.push(tabs); if tabs > 0 {
for c in leading.chars() {
if c == '\t' {
out.push_str(" ");
} else {
out.push(c);
}
}
} else {
out.push_str(leading);
}
out.push_str(rest);
}
TAB_ADJUST.with(|t| *t.borrow_mut() = Some(adjust));
out
}
fn strip_trailing_spaces(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for (i, line) in s.split('\n').enumerate() {
if i > 0 {
out.push('\n');
}
out.push_str(line.trim_end_matches(' '));
}
out
}
fn parse_core_generic<B: Builder, const CHECK_DUPLICATES: bool>(
front_matter: &str,
strict: bool,
) -> Result<B::Mapping, LimaError> {
if front_matter.len() > DOCUMENT_SIZE_LIMIT {
return Err(LimaError::new(
Code::ResourceLimit,
1,
format!("Lima: document exceeds maximum size of {DOCUMENT_SIZE_LIMIT} bytes at line 1"),
));
}
TAB_ADJUST.with(|t| *t.borrow_mut() = None);
let mut owned = front_matter.to_string();
if owned.contains('\r') {
owned = owned.replace("\r\n", "\n").replace('\r', "\n");
}
if owned.contains('\t') {
owned = expand_leading_tabs(&owned);
}
if owned.contains(" \n") || owned.ends_with(' ') {
owned = strip_trailing_spaces(&owned);
}
let front_matter: &str = &owned;
if front_matter.is_empty() {
return Ok(B::m_create());
}
if strict {
let mut search_from = 0usize;
loop {
let line_end = front_matter[search_from..]
.find('\n')
.map(|p| search_from + p);
let line = &front_matter[search_from..line_end.unwrap_or(front_matter.len())];
if has_space_before_colon(line) {
let l = line_at(front_matter, search_from);
return Err(LimaError::new(
Code::InvalidQuote,
l,
format!("Lima: space between closing quote and colon at line {l}"),
));
}
match line_end {
Some(e) => search_from = e + 1,
None => break,
}
}
}
let top_keys = scan_top_keys(front_matter, strict)?;
if top_keys.len() > TOP_LEVEL_KEY_LIMIT {
return Err(LimaError::new(
Code::ResourceLimit,
1,
format!("Lima: too many top-level key entries (max {TOP_LEVEL_KEY_LIMIT}) at line 1"),
));
}
if strict {
let first = top_keys
.first()
.map_or(front_matter.len(), |tk| tk.key_start);
check_range_lines(front_matter, 0, first, 1, RangeOwner::None)?;
}
let mut root: B::Mapping = B::m_create();
let mut depth_risk = false;
let mut top_keys = top_keys.into_iter().peekable();
while let Some(tk) = top_keys.next() {
let next_start = top_keys
.peek()
.map(|n| n.key_start)
.unwrap_or(front_matter.len());
check_key_length(&tk.key, tk.line)?;
if CHECK_DUPLICATES {
check_duplicate_key(B::m_has_key(&root, &tk.key), &tk.key, tk.line, strict)?;
}
let span_end = next_start.min(front_matter.len());
let (value_start, block_start) = if tk.is_block {
(tk.value_start, Some(tk.value_start))
} else {
let value_start = inline_value_start(front_matter, tk.value_start, span_end);
(
value_start,
bare_key_block_start(front_matter, value_start, span_end),
)
};
let mut inline_line: Option<(usize, bool)> = None;
let value = if let Some(block_start) = block_start {
parse_block_range_checked::<B, CHECK_DUPLICATES>(
front_matter,
block_start,
next_start,
strict,
tk.line + 1,
&mut depth_risk,
)?
.unwrap_or(B::v_null(tk.line))
} else {
let first_newline = front_matter[value_start..span_end]
.find('\n')
.map(|p| p + value_start);
let value_col = if B::POSITIONED {
let value_line_start = front_matter[..value_start].rfind('\n').map_or(0, |p| p + 1);
front_matter[value_line_start..value_start].chars().count()
} else {
0
};
let inline_src = |raw: &str| {
B::POSITIONED.then(|| ReferenceSource {
raw: Some(physical_raw(raw)),
line: tk.line,
col: value_col,
tab_adjust: tab_adjust_slice(tk.line, 1),
})
};
let first_line = &front_matter[value_start..first_newline.unwrap_or(span_end)];
let marker = first_line.trim_matches(is_trim_whitespace) == "|";
inline_line = first_newline.map(|nl| (nl, marker));
if marker {
let body: Vec<&str> = match first_newline {
Some(nl) => front_matter[nl + 1..span_end].split('\n').collect(),
None => Vec::new(),
};
let (joined, raw_body, consumed) = build_block_scalar(&body, 0, tk.line)?;
B::v_string_src(
joined,
tk.line + 1,
ReferenceSource {
raw: Some(raw_body),
line: tk.line + 1,
col: 0,
tab_adjust: tab_adjust_slice(tk.line + 1, consumed),
},
)
} else if first_line.contains('#') {
let stripped = strip_comment(first_line);
parse_flow_or_scalar_value_checked::<B, CHECK_DUPLICATES>(
&stripped,
strict,
tk.line,
inline_src(first_line),
)?
} else {
parse_flow_or_scalar_value_checked::<B, CHECK_DUPLICATES>(
first_line.trim_end_matches(is_trim_whitespace),
strict,
tk.line,
inline_src(first_line),
)?
}
};
if strict {
if let Some(block_start) = block_start {
check_range_lines(
front_matter,
block_start,
next_start,
tk.line + 1,
RangeOwner::Mapping,
)?;
} else if let Some((line_end, marker)) =
inline_line.filter(|&(nl, _)| nl + 1 < next_start)
{
let owner = if marker {
RangeOwner::Scalar
} else {
RangeOwner::Inline
};
check_range_lines(front_matter, line_end + 1, next_start, tk.line + 1, owner)?;
}
}
B::m_set(&mut root, tk.key, value);
}
if depth_risk && B::m_max_depth(&root) > NESTING_DEPTH_LIMIT as u32 {
return Err(LimaError::new(
Code::ResourceLimit,
1,
format!("Lima: nesting depth exceeds maximum of {NESTING_DEPTH_LIMIT} at line 1"),
));
}
Ok(root)
}
#[derive(Default)]
pub struct CoreOptions {
pub strict: bool,
pub on_warning: Option<Box<dyn FnMut(Diagnostic)>>,
}
impl From<bool> for CoreOptions {
fn from(strict: bool) -> Self {
Self {
strict,
on_warning: None,
}
}
}
pub fn parse_core(
front_matter: &str,
options: impl Into<CoreOptions>,
) -> Result<LimaValue, LimaError> {
let mut options = options.into();
if options.on_warning.is_none() {
return if options.strict {
parse_core_generic::<PlainBuilder, true>(front_matter, true)
} else {
parse_core_generic::<PlainBuilder, false>(front_matter, false)
}
.map(LimaValue::Mapping);
}
begin_warning_collection(true);
let result = parse_core_generic::<PlainBuilder, true>(front_matter, options.strict)
.map(LimaValue::Mapping);
let warnings = finish_warning_collection();
if let Some(callback) = options.on_warning.as_mut() {
for warning in warnings {
callback(warning);
}
}
result
}
pub fn parse_core_with_positions(
front_matter: &str,
strict: bool,
) -> Result<Vec<(String, PositionedValue)>, LimaError> {
if strict {
parse_core_generic::<PositionedBuilder, true>(front_matter, true)
} else {
parse_core_generic::<PositionedBuilder, false>(front_matter, false)
}
}
pub(crate) fn parse_core_with_positions_options(
front_matter: &str,
strict: bool,
collect_warnings: bool,
) -> Result<Vec<(String, PositionedValue)>, LimaError> {
if strict || collect_warnings {
parse_core_generic::<PositionedBuilder, true>(front_matter, strict)
} else {
parse_core_generic::<PositionedBuilder, false>(front_matter, false)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn get<'a>(v: &'a LimaValue, key: &str) -> &'a LimaValue {
let LimaValue::Mapping(entries) = v else {
panic!("expected mapping")
};
&entries
.iter()
.find(|(k, _)| k == key)
.unwrap_or_else(|| panic!("missing key {key:?} in {entries:?}"))
.1
}
#[test]
fn flat_document() {
let v = parse_core("title: Hello\ncount: 3\n", false).unwrap();
assert_eq!(get(&v, "title"), &LimaValue::String("Hello".into()));
assert_eq!(get(&v, "count"), &LimaValue::Int(3));
}
#[test]
fn nested_mapping() {
let v = parse_core("address:\n street: Main St\n city: Springfield\n", false).unwrap();
let addr = get(&v, "address");
assert_eq!(get(addr, "street"), &LimaValue::String("Main St".into()));
assert_eq!(get(addr, "city"), &LimaValue::String("Springfield".into()));
}
#[test]
fn block_sequence() {
let v = parse_core("tags:\n - a\n - b\n - c\n", false).unwrap();
let LimaValue::Array(items) = get(&v, "tags") else {
panic!("expected array")
};
assert_eq!(
items,
&[
LimaValue::String("a".into()),
LimaValue::String("b".into()),
LimaValue::String("c".into()),
]
);
}
#[test]
fn sequence_of_mappings() {
let v = parse_core(
"items:\n - name: a\n qty: 1\n - name: b\n qty: 2\n",
false,
)
.unwrap();
let LimaValue::Array(items) = get(&v, "items") else {
panic!("expected array")
};
assert_eq!(get(&items[0], "name"), &LimaValue::String("a".into()));
assert_eq!(get(&items[0], "qty"), &LimaValue::Int(1));
assert_eq!(get(&items[1], "name"), &LimaValue::String("b".into()));
}
#[test]
fn flow_sequence_and_mapping() {
let v = parse_core("tags: [a, b, c]\nauthor: {name: Alice, age: 30}\n", false).unwrap();
let LimaValue::Array(tags) = get(&v, "tags") else {
panic!("expected array")
};
assert_eq!(tags.len(), 3);
let author = get(&v, "author");
assert_eq!(get(author, "name"), &LimaValue::String("Alice".into()));
assert_eq!(get(author, "age"), &LimaValue::Int(30));
}
#[test]
fn duplicate_key_strict_throws() {
assert!(parse_core("title: First\ntitle: Second", true).is_err());
}
#[test]
fn duplicate_key_non_strict_last_wins() {
let v = parse_core("title: First\ntitle: Second", false).unwrap();
assert_eq!(get(&v, "title"), &LimaValue::String("Second".into()));
}
#[test]
fn comments_are_stripped() {
let v = parse_core(
"title: Hello # a comment\n# full line comment\ncount: 3\n",
false,
)
.unwrap();
assert_eq!(get(&v, "title"), &LimaValue::String("Hello".into()));
assert_eq!(get(&v, "count"), &LimaValue::Int(3));
}
#[test]
fn unrecognized_top_level_lines_are_skipped_non_strict_and_throw_strict() {
let input = "title: Hello\nthis is not a key line at all\ncount: 42";
let v = parse_core(input, false).unwrap();
assert_eq!(get(&v, "title"), &LimaValue::String("Hello".into()));
assert_eq!(get(&v, "count"), &LimaValue::Int(42));
let error = parse_core(input, true).unwrap_err();
assert_eq!(error.code, Code::UnrecognizedLine);
assert_eq!(error.line, Some(2));
}
#[test]
fn indented_line_is_not_a_top_level_key() {
let v = parse_core(" bogus: value\nreal: Hello", false).unwrap();
let LimaValue::Mapping(entries) = &v else {
panic!()
};
assert_eq!(entries.len(), 1);
assert_eq!(get(&v, "real"), &LimaValue::String("Hello".into()));
}
#[test]
fn block_scalar_basic() {
let v = parse_core("description: |\n Line one.\n Line two.\n", false).unwrap();
assert_eq!(
get(&v, "description"),
&LimaValue::String("Line one.\nLine two.".into())
);
}
#[test]
fn block_scalar_continuation() {
let v = parse_core(
"description: |\n This is long\n ^^and continues.\n",
false,
)
.unwrap();
assert_eq!(
get(&v, "description"),
&LimaValue::String("This is long and continues.".into())
);
}
#[test]
fn block_scalar_trailing_blank_lines_stripped() {
let v = parse_core("description: |\n Line one.\n\n\nnext: value\n", false).unwrap();
assert_eq!(
get(&v, "description"),
&LimaValue::String("Line one.".into())
);
assert_eq!(get(&v, "next"), &LimaValue::String("value".into()));
}
#[test]
fn quoted_key_with_escaped_quote() {
let v = parse_core(r#""say \"hi\"": value"#, false).unwrap();
assert_eq!(get(&v, "say \"hi\""), &LimaValue::String("value".into()));
}
#[test]
fn space_before_colon_strict_error() {
assert!(parse_core("\"first name\" : Alice", true).is_err());
}
#[test]
fn core_12_unicode_keys_and_message_hints() {
let v = parse_core("größe: one\n𠀀: two\nautor*in: three\n", false).unwrap();
assert_eq!(get(&v, "größe"), &LimaValue::String("one".into()));
assert_eq!(get(&v, "𠀀"), &LimaValue::String("two".into()));
assert_eq!(get(&v, "autor*in"), &LimaValue::String("three".into()));
let whitespace = parse_core("\u{a0}key: value\n", true).unwrap_err();
assert!(whitespace
.message
.contains("U+00A0 before the key; Lima indents by spaces only"));
let colon = parse_core("title: value\n", true).unwrap_err();
assert!(colon.message.contains(
"the key-value separator is \": \" (U+003A and a space), not U+FF1A FULLWIDTH COLON"
));
}
#[test]
fn spaced_key_with_separator_names_the_quoted_key_at_every_level() {
for input in ["x y: 2\n", "outer:\n x y: 2\n"] {
let error = parse_core(input, true).unwrap_err();
assert!(error
.message
.contains("(quote a key that holds spaces: 'x y':)"));
assert!(!error.message.contains("a key needs \": \" after it"));
}
let error = parse_core("x/y: 2\n", true).unwrap_err();
assert!(error
.message
.contains("an unquoted key starts with a Unicode letter or number"));
}
}