use super::ast::AstNode as _; use super::convert::{
DescendantsWalkResult, TopLevelWalkResult, classify_recovery_error, decode_string_token,
is_comment_kind, is_trivia_kind, number_meta_value, parse_date_token, parse_decimal_token,
};
use rowan::{Language, NodeOrToken};
use rustledger_core::cost::{CostNumber, CostSpec};
use rustledger_core::directive::{PriceAnnotation, PriceKind};
use rustledger_core::{
Account, Amount, Currency, IncompleteAmount, InternedStr, Link, MetaValue, Metadata, NaiveDate,
Posting, Span, Spanned, Tag,
};
#[allow(dead_code)]
pub(super) fn top_level_node_spans(
root: &crate::SyntaxNode,
bom_offset: u32,
) -> Vec<(crate::SyntaxKind, Span)> {
let green = root.green();
let mut out = Vec::new();
let mut offset = bom_offset as usize;
for child in green.children() {
let len = match &child {
NodeOrToken::Node(n) => u32::from(n.text_len()) as usize,
NodeOrToken::Token(t) => u32::from(t.text_len()) as usize,
};
if let NodeOrToken::Node(n) = child {
let kind = crate::BeancountLanguage::kind_from_raw(n.kind());
out.push((kind, Span::new(offset, offset + len)));
}
offset += len;
}
out
}
fn flag_char(kind: crate::SyntaxKind, text: &str) -> Option<char> {
use crate::SyntaxKind as K;
match kind {
K::STAR | K::TXN_KW => Some('*'),
K::PENDING_KW => Some('!'),
K::HASH => Some('#'),
K::FLAG => text.chars().next(),
K::CURRENCY if text.len() == 1 => text.chars().next(),
_ => None,
}
}
pub(super) fn convert_transaction_header(
node: &rowan::GreenNodeData,
base: usize,
) -> Option<(rustledger_core::directive::Transaction, Span)> {
use crate::SyntaxKind as K;
let span = Span::new(base, base + u32::from(node.text_len()) as usize);
let mut date: Option<NaiveDate> = None;
let mut date_seen = false;
let mut flag = '*';
let mut seen_flag = false;
let mut seen_str_tag_link = false;
let mut strings: Vec<String> = Vec::new();
let mut tags: Vec<Tag> = Vec::new();
let mut links: Vec<Link> = Vec::new();
let mut past_header = false;
for child in node.children() {
let NodeOrToken::Token(t) = child else {
continue;
};
let kind = crate::BeancountLanguage::kind_from_raw(t.kind());
let text = t.text();
if past_header {
match kind {
K::TAG => {
let tg = Tag::new(text.trim_start_matches('#'));
if !tags.contains(&tg) {
tags.push(tg);
}
}
K::LINK => {
let lk = Link::new(text.trim_start_matches('^'));
if !links.contains(&lk) {
links.push(lk);
}
}
_ => {}
}
} else {
match kind {
K::NEWLINE => past_header = true,
K::DATE if !date_seen => {
date_seen = true;
date = parse_date_token(text);
}
K::STRING => {
seen_str_tag_link = true;
if let Some(s) = decode_string_token(text) {
strings.push(s);
}
}
K::TAG => {
seen_str_tag_link = true;
tags.push(Tag::new(text.trim_start_matches('#')));
}
K::LINK => {
seen_str_tag_link = true;
links.push(Link::new(text.trim_start_matches('^')));
}
k if !seen_flag && !seen_str_tag_link => {
if let Some(c) = flag_char(k, text) {
flag = c;
seen_flag = true;
}
}
_ => {}
}
}
}
let date = date?;
let mut it = strings.into_iter();
let (payee_str, narration_str) = match (it.next(), it.next(), it.next()) {
(None, _, _) => (None, String::new()),
(Some(n), None, _) => (None, n),
(Some(p), Some(n), None) => (Some(p), n),
(Some(_), Some(_), Some(c)) => (None, it.last().unwrap_or(c)),
};
let txn = rustledger_core::directive::Transaction {
date,
flag,
payee: payee_str.map(InternedStr::from),
narration: InternedStr::from(narration_str),
tags,
links,
meta: Metadata::default(),
postings: Vec::new(),
trailing_comments: Vec::new(),
};
Some((txn, span))
}
fn simple_amount(node: &rowan::GreenNodeData) -> Option<IncompleteAmount> {
use crate::SyntaxKind as K;
let mut sign_minus = false;
let mut number: Option<rust_decimal::Decimal> = None;
let mut number_seen = false;
let mut currency: Option<Currency> = None;
let mut complex = false;
for child in node.children() {
let NodeOrToken::Token(t) = child else {
complex = true;
continue;
};
match crate::BeancountLanguage::kind_from_raw(t.kind()) {
K::MINUS if !number_seen => sign_minus = true,
K::PLUS if !number_seen => {}
K::NUMBER if !number_seen => {
number_seen = true;
number = parse_decimal_token(t.text());
if number.is_none() {
complex = true;
}
}
K::CURRENCY if currency.is_none() => currency = Some(Currency::new(t.text())),
K::WHITESPACE => {}
K::NUMBER
| K::PLUS
| K::MINUS
| K::STAR
| K::SLASH
| K::L_PAREN
| K::R_PAREN
| K::CURRENCY => complex = true,
_ => {}
}
}
if complex {
return None;
}
let number = number.map(|n| if sign_minus { -n } else { n });
match (number, currency) {
(Some(n), Some(c)) => Some(IncompleteAmount::Complete(Amount::new(n, c))),
(Some(n), None) => Some(IncompleteAmount::NumberOnly(n)),
(None, Some(c)) => Some(IncompleteAmount::CurrencyOnly(c)),
(None, None) => None,
}
}
fn convert_cost_spec(node: &rowan::GreenNodeData) -> CostSpec {
use crate::SyntaxKind as K;
let mut is_total = false;
let mut first_number: Option<rust_decimal::Decimal> = None;
let mut seen_number = false;
let mut past_hash = false;
let mut post_hash_seen = false;
let mut post_hash_total: Option<rust_decimal::Decimal> = None;
let mut currency: Option<Currency> = None;
let mut date: Option<NaiveDate> = None;
let mut date_seen = false;
let mut label: Option<String> = None;
let mut label_seen = false;
for child in node.children() {
let NodeOrToken::Token(t) = child else {
continue;
};
match crate::BeancountLanguage::kind_from_raw(t.kind()) {
K::L_DOUBLE_BRACE => is_total = true,
K::NUMBER => {
if past_hash {
if !post_hash_seen {
post_hash_seen = true;
post_hash_total = parse_decimal_token(t.text());
}
} else if !seen_number {
seen_number = true;
first_number = parse_decimal_token(t.text());
}
}
K::HASH => {
if seen_number {
past_hash = true;
}
}
K::CURRENCY if currency.is_none() => {
currency = Some(Currency::new(t.text()));
}
K::DATE if !date_seen => {
date_seen = true;
date = parse_date_token(t.text());
}
K::STRING if !label_seen => {
label_seen = true;
label = decode_string_token(t.text());
}
_ => {}
}
}
let number = if let Some(total) = post_hash_total {
Some(CostNumber::Total { value: total })
} else {
match (first_number, is_total) {
(Some(v), true) => Some(CostNumber::Total { value: v }),
(Some(v), false) => Some(CostNumber::PerUnit { value: v }),
(None, _) => None,
}
};
CostSpec {
number,
currency,
date,
label,
merge: cost_is_merge(node),
}
}
fn cost_is_merge(node: &rowan::GreenNodeData) -> bool {
use crate::SyntaxKind as K;
let mut past_opener = false;
for child in node.children() {
let NodeOrToken::Token(t) = child else {
continue;
};
match crate::BeancountLanguage::kind_from_raw(t.kind()) {
K::L_BRACE | K::L_DOUBLE_BRACE | K::L_BRACE_HASH => past_opener = true,
K::WHITESPACE if past_opener => {}
K::STAR if past_opener => return true,
_ if past_opener => return false,
_ => {}
}
}
false
}
fn convert_price_annotation(node: &rowan::GreenNodeData) -> Option<PriceAnnotation> {
use crate::SyntaxKind as K;
let mut is_total = false;
let mut amount_present = false;
let mut amount: Option<IncompleteAmount> = None;
for child in node.children() {
match &child {
NodeOrToken::Token(t) => {
if crate::BeancountLanguage::kind_from_raw(t.kind()) == K::AT_AT {
is_total = true;
}
}
NodeOrToken::Node(n) => {
if crate::BeancountLanguage::kind_from_raw(n.kind()) == K::AMOUNT && !amount_present
{
amount_present = true;
amount = simple_amount(n);
}
}
}
}
if amount_present && amount.is_none() {
return None; }
Some(PriceAnnotation {
kind: if is_total {
PriceKind::Total
} else {
PriceKind::Unit
},
amount,
})
}
fn meta_value(entry: &rowan::GreenNodeData) -> MetaValue {
use crate::SyntaxKind as K;
let mut string_t: Option<String> = None;
let mut number_t: Option<String> = None;
let mut currency_t: Option<String> = None;
let mut date_t: Option<String> = None;
let mut account_t: Option<String> = None;
let mut bool_v: Option<bool> = None;
let mut tag_link: Option<MetaValue> = None;
let mut past_key = false;
let mut minus = false;
let mut minus_decided = false;
for child in entry.children() {
let NodeOrToken::Token(t) = child else {
continue;
};
let kind = crate::BeancountLanguage::kind_from_raw(t.kind());
match kind {
K::STRING if string_t.is_none() => string_t = Some(t.text().to_string()),
K::NUMBER if number_t.is_none() => number_t = Some(t.text().to_string()),
K::CURRENCY if currency_t.is_none() => currency_t = Some(t.text().to_string()),
K::DATE if date_t.is_none() => date_t = Some(t.text().to_string()),
K::ACCOUNT if account_t.is_none() => account_t = Some(t.text().to_string()),
K::BOOL_TRUE if bool_v.is_none() => bool_v = Some(true),
K::BOOL_FALSE if bool_v.is_none() => bool_v = Some(false),
K::TAG if tag_link.is_none() => {
tag_link = Some(MetaValue::Tag(Tag::new(t.text().trim_start_matches('#'))));
}
K::LINK if tag_link.is_none() => {
tag_link = Some(MetaValue::Link(Link::new(t.text().trim_start_matches('^'))));
}
_ => {}
}
if past_key && !minus_decided {
match kind {
K::MINUS => {
minus = true;
minus_decided = true;
}
K::NUMBER => minus_decided = true,
_ => {}
}
}
if kind == K::META_KEY {
past_key = true;
}
}
if let Some(s) = string_t
&& let Some(decoded) = decode_string_token(&s)
{
return MetaValue::String(decoded);
}
if let Some(nt) = number_t
&& let Some(mut dec) = parse_decimal_token(&nt)
{
if minus {
dec = -dec;
}
if let Some(c) = currency_t {
return MetaValue::Amount(Amount::new(dec, Currency::new(&c)));
}
return number_meta_value(&nt, dec);
}
if let Some(dt) = date_t
&& let Some(date) = parse_date_token(&dt)
{
return MetaValue::Date(date);
}
if let Some(a) = account_t {
return MetaValue::Account(Account::new(&a));
}
if let Some(c) = currency_t {
return MetaValue::Currency(Currency::new(&c));
}
if let Some(b) = bool_v {
return MetaValue::Bool(b);
}
if let Some(tl) = tag_link {
return tl;
}
MetaValue::None
}
fn meta_entry_kv(entry: &rowan::GreenNodeData) -> Option<(String, MetaValue)> {
use crate::SyntaxKind as K;
let key = entry.children().find_map(|c| match c {
NodeOrToken::Token(t)
if crate::BeancountLanguage::kind_from_raw(t.kind()) == K::META_KEY =>
{
Some(t.text().strip_suffix(':').unwrap_or(t.text()).to_string())
}
_ => None,
})?;
Some((key, meta_value(entry)))
}
fn posting_flag_char(kind: crate::SyntaxKind, text: &str) -> Option<char> {
use crate::SyntaxKind as K;
match kind {
K::STAR => Some('*'),
K::PENDING_KW => Some('!'),
K::HASH => Some('#'),
K::FLAG => text.chars().next(),
K::CURRENCY if text.len() == 1 => text.chars().next(),
_ => None,
}
}
pub(super) fn convert_simple_posting(
node: &rowan::GreenNodeData,
base: usize,
) -> Option<Spanned<Posting>> {
use crate::SyntaxKind as K;
let mut flag: Option<char> = None;
let mut flag_decided = false;
let mut account: Option<Account> = None;
let mut units: Option<IncompleteAmount> = None;
let mut cost: Option<CostSpec> = None;
let mut price: Option<PriceAnnotation> = None;
let mut meta = Metadata::default();
let mut trailing_comments: Vec<String> = Vec::new();
let mut newline_off: Option<usize> = None;
let mut amount_seen = false;
let mut offset = base;
for child in node.children() {
let len = match &child {
NodeOrToken::Node(n) => u32::from(n.text_len()) as usize,
NodeOrToken::Token(t) => u32::from(t.text_len()) as usize,
};
match &child {
NodeOrToken::Token(t) => {
let kind = crate::BeancountLanguage::kind_from_raw(t.kind());
if kind == K::ACCOUNT && account.is_none() {
flag_decided = true; account = Some(Account::new(t.text()));
} else if kind == K::NEWLINE && newline_off.is_none() {
newline_off = Some(offset);
} else if newline_off.is_none() && is_comment_kind(kind) {
trailing_comments.push(t.text().to_string());
} else if !flag_decided && account.is_none() && kind != K::WHITESPACE {
flag_decided = true;
flag = posting_flag_char(kind, t.text());
}
}
NodeOrToken::Node(n) => match crate::BeancountLanguage::kind_from_raw(n.kind()) {
K::AMOUNT if !amount_seen => {
amount_seen = true;
units = Some(simple_amount(n)?);
}
K::COST_SPEC if cost.is_none() => cost = Some(convert_cost_spec(n)),
K::PRICE_ANNOTATION if price.is_none() => {
price = Some(convert_price_annotation(n)?);
}
K::META_ENTRY => {
if let Some((k, v)) = meta_entry_kv(n) {
meta.insert(k, v);
}
}
K::AMOUNT => return None,
_ => {}
},
}
offset += len;
}
let account = account?;
let end = newline_off.unwrap_or(base + u32::from(node.text_len()) as usize);
Some(Spanned::new(
Posting {
account,
units,
cost,
price,
flag,
meta,
comments: Vec::new(),
trailing_comments,
},
Span::new(base, end),
))
}
pub(super) fn convert_transaction(
node: &rowan::GreenNodeData,
base: usize,
) -> Option<Spanned<rustledger_core::Directive>> {
use crate::SyntaxKind as K;
let (mut txn, span) = convert_transaction_header(node, base)?;
let mut postings = Vec::new();
let mut meta = Metadata::default();
let mut offset = base;
for child in node.children() {
let len = match &child {
NodeOrToken::Node(n) => u32::from(n.text_len()) as usize,
NodeOrToken::Token(t) => u32::from(t.text_len()) as usize,
};
match &child {
NodeOrToken::Token(t) => {
let kind = crate::BeancountLanguage::kind_from_raw(t.kind());
if is_comment_kind(kind) || kind == K::PIPE {
return None;
}
}
NodeOrToken::Node(n) => match crate::BeancountLanguage::kind_from_raw(n.kind()) {
K::POSTING => postings.push(convert_simple_posting(n, offset)?),
K::META_ENTRY => {
if let Some((k, v)) = meta_entry_kv(n) {
meta.insert(k, v);
}
}
_ => {}
},
}
offset += len;
}
txn.postings = postings;
txn.meta = meta;
Some(Spanned::new(
rustledger_core::Directive::Transaction(txn),
span,
))
}
pub(super) fn walk_descendants(
root: &crate::SyntaxNode,
bom_offset: u32,
collect_occurrences: bool,
) -> DescendantsWalkResult {
let mut w = DescendantsWalker {
offset: bom_offset as usize,
preceded_by_ws: false,
collect_occurrences,
result: DescendantsWalkResult {
inline_errors: Vec::new(),
top_level_comments: Vec::new(),
currency_occurrences: Vec::new(),
account_occurrences: Vec::new(),
},
};
w.walk(&root.green(), false);
w.result
}
struct DescendantsWalker {
offset: usize,
preceded_by_ws: bool,
collect_occurrences: bool,
result: DescendantsWalkResult,
}
impl DescendantsWalker {
fn walk(&mut self, node: &rowan::GreenNodeData, in_error_node: bool) {
use crate::SyntaxKind as K;
for child in node.children() {
match child {
NodeOrToken::Node(n) => {
let kind = crate::BeancountLanguage::kind_from_raw(n.kind());
if super::ast::Directive::can_cast(kind) {
self.preceded_by_ws = false;
}
self.walk(n, in_error_node || kind == K::ERROR_NODE);
}
NodeOrToken::Token(t) => {
let start = self.offset;
let len = u32::from(t.text_len()) as usize;
self.offset += len;
let kind = crate::BeancountLanguage::kind_from_raw(t.kind());
self.token(kind, t.text(), start, len, in_error_node);
}
}
}
}
fn token(
&mut self,
kind: crate::SyntaxKind,
text: &str,
start: usize,
len: usize,
in_error_node: bool,
) {
use crate::SyntaxKind as K;
match kind {
K::NEWLINE => self.preceded_by_ws = false,
K::WHITESPACE => self.preceded_by_ws = true,
k if is_comment_kind(k) => {
if !self.preceded_by_ws {
self.result.top_level_comments.push(Spanned::new(
text.to_string(),
Span::new(start, start + len),
));
}
}
_ => self.preceded_by_ws = false,
}
if kind == K::BOM {
return;
}
let has_bom = text.contains(crate::bom::BOM_CHAR);
let is_error_token = kind == K::ERROR_TOKEN;
let needs = (self.collect_occurrences && matches!(kind, K::CURRENCY | K::ACCOUNT))
|| has_bom
|| is_error_token;
if !needs {
return;
}
let span = Span::new(start, start + len);
if self.collect_occurrences && kind == K::CURRENCY && !in_error_node {
self.result
.currency_occurrences
.push(Spanned::new(Currency::new(text), span));
}
if self.collect_occurrences && kind == K::ACCOUNT && !in_error_node {
self.result
.account_occurrences
.push(Spanned::new(Account::new(text), span));
}
if (!has_bom && !is_error_token) || in_error_node {
return;
}
if has_bom {
self.result.inline_errors.push(
crate::ParseError::new(crate::ParseErrorKind::BomInDirectiveBody, span)
.with_hint(crate::diagnostics::BOM_REMOVAL_HINT),
);
} else {
self.result.inline_errors.push(crate::ParseError::new(
crate::ParseErrorKind::SyntaxError("unexpected input".to_string()),
span,
));
}
}
}
fn child_len(child: NodeOrToken<&rowan::GreenNodeData, &rowan::GreenTokenData>) -> usize {
match child {
NodeOrToken::Node(n) => u32::from(n.text_len()) as usize,
NodeOrToken::Token(t) => u32::from(t.text_len()) as usize,
}
}
pub(super) fn walk_top_level(
root: &crate::SyntaxNode,
stripped: &str,
bom_offset: u32,
) -> TopLevelWalkResult {
use crate::SyntaxKind as K;
let mut errors = Vec::new();
let mut section_marker_comments = Vec::new();
let green = root.green();
let mut offset = 0usize;
for child in green.children() {
let len = child_len(child);
if let NodeOrToken::Node(n) = &child {
let kind = crate::BeancountLanguage::kind_from_raw(n.kind());
if super::ast::Directive::can_cast(kind) {
tl_indented_check(n, offset, bom_offset, stripped, &mut errors);
}
match kind {
K::CUSTOM_DIRECTIVE => tl_custom_check(n, offset, bom_offset, &mut errors),
K::TRANSACTION => tl_transaction_body_check(n, offset, bom_offset, &mut errors),
K::ERROR_NODE => {
tl_error_node_check(n, offset, bom_offset, stripped, &mut errors);
tl_section_marker_check(n, offset, bom_offset, &mut section_marker_comments);
}
_ => {}
}
}
offset += len;
}
TopLevelWalkResult {
errors,
section_marker_comments,
}
}
fn tl_indented_check(
node: &rowan::GreenNodeData,
base: usize,
bom_offset: u32,
stripped: &str,
out: &mut Vec<crate::ParseError>,
) {
let mut offset = base;
let mut content: Option<(usize, usize)> = None;
for child in node.children() {
let len = child_len(child);
if let NodeOrToken::Token(t) = &child
&& !is_trivia_kind(crate::BeancountLanguage::kind_from_raw(t.kind()))
{
content = Some((offset, offset + len));
break;
}
offset += len;
}
let Some((content_start, content_end)) = content else {
return;
};
let line_start = stripped
.as_bytes()
.get(..content_start)
.and_then(|bytes| bytes.iter().rposition(|&b| b == b'\n'))
.map_or(0, |nl| nl + 1);
if content_start > line_start {
let span = Span::new(
line_start + bom_offset as usize,
content_end + bom_offset as usize,
);
out.push(crate::ParseError::new(
crate::ParseErrorKind::SyntaxError(
"top-level directive must start at column 0".to_string(),
),
span,
));
}
}
fn tl_custom_check(
node: &rowan::GreenNodeData,
base: usize,
bom_offset: u32,
out: &mut Vec<crate::ParseError>,
) {
use crate::SyntaxKind as K;
let mut toks: Vec<(crate::SyntaxKind, usize, usize)> = Vec::new();
let mut offset = base;
for child in node.children() {
let len = child_len(child);
if let NodeOrToken::Token(t) = &child {
let kind = crate::BeancountLanguage::kind_from_raw(t.kind());
if !is_trivia_kind(kind) {
toks.push((kind, offset, len));
}
}
offset += len;
}
let mut seen_type_string = false;
for i in 0..toks.len() {
let (kind, start, len) = toks[i];
if !seen_type_string {
if kind == K::STRING {
seen_type_string = true;
}
continue;
}
if kind == K::CURRENCY && !(i > 0 && toks[i - 1].0 == K::NUMBER) {
let span = Span::new(
start + bom_offset as usize,
start + len + bom_offset as usize,
);
out.push(crate::ParseError::new(
crate::ParseErrorKind::SyntaxError(
"bare currency literal is not a valid custom directive value".to_string(),
),
span,
));
}
}
}
fn tl_transaction_body_check(
node: &rowan::GreenNodeData,
base: usize,
bom_offset: u32,
out: &mut Vec<crate::ParseError>,
) {
use crate::SyntaxKind as K;
let mut past_header = false;
let mut saw_header_content = false;
let mut line_start: Option<usize> = None;
let mut line_has_content = false;
let mut offset = base;
for child in node.children() {
let len = child_len(child);
match &child {
NodeOrToken::Token(t) => {
let kind = crate::BeancountLanguage::kind_from_raw(t.kind());
let (start, end) = (offset, offset + len);
if past_header {
if line_start.is_none() {
line_start = Some(start);
}
if kind == K::NEWLINE {
if line_has_content && let Some(ls) = line_start {
let span =
Span::new(ls + bom_offset as usize, end + bom_offset as usize);
out.push(crate::ParseError::new(
crate::ParseErrorKind::SyntaxError("unexpected input".to_string()),
span,
));
}
line_start = None;
line_has_content = false;
} else if !is_trivia_kind(kind)
&& !is_comment_kind(kind)
&& !matches!(kind, K::TAG | K::LINK)
{
line_has_content = true;
}
} else if kind == K::NEWLINE {
if saw_header_content {
past_header = true;
}
} else if !is_trivia_kind(kind) {
saw_header_content = true;
}
}
NodeOrToken::Node(_) => {
line_start = None;
line_has_content = false;
past_header = true;
}
}
offset += len;
}
}
fn tl_error_node_check(
node: &rowan::GreenNodeData,
base: usize,
bom_offset: u32,
stripped: &str,
out: &mut Vec<crate::ParseError>,
) {
use crate::SyntaxKind as K;
let mut line_start: Option<usize> = None;
let mut first_non_trivia: Option<crate::SyntaxKind> = None;
let mut offset = base;
for child in node.children() {
let len = child_len(child);
if let NodeOrToken::Token(t) = &child {
let kind = crate::BeancountLanguage::kind_from_raw(t.kind());
let (start, end) = (offset, offset + len);
if line_start.is_none() {
line_start = Some(start);
}
if kind == K::NEWLINE {
let is_section = first_non_trivia == Some(K::STAR);
let is_comment = matches!(first_non_trivia, Some(k) if is_comment_kind(k));
if !is_section
&& !is_comment
&& first_non_trivia.is_some()
&& let Some(ls) = line_start
{
let span = Span::new(ls + bom_offset as usize, end + bom_offset as usize);
let line_text = stripped.get(ls..end).unwrap_or("");
let primary = classify_recovery_error(line_text, span);
let primary_is_bom =
matches!(primary.kind, crate::ParseErrorKind::BomInDirectiveBody);
out.push(primary);
if !primary_is_bom && line_text.contains(crate::bom::BOM_CHAR) {
out.push(
crate::ParseError::new(crate::ParseErrorKind::BomInDirectiveBody, span)
.with_hint(crate::diagnostics::BOM_REMOVAL_HINT),
);
}
}
line_start = None;
first_non_trivia = None;
} else if first_non_trivia.is_none() && !is_trivia_kind(kind) {
first_non_trivia = Some(kind);
}
}
offset += len;
}
}
fn tl_section_marker_check(
node: &rowan::GreenNodeData,
base: usize,
bom_offset: u32,
out: &mut Vec<Spanned<String>>,
) {
use crate::SyntaxKind as K;
let mut line_start: Option<usize> = None;
let mut first_non_trivia: Option<crate::SyntaxKind> = None;
let mut offset = base;
for child in node.children() {
let len = child_len(child);
if let NodeOrToken::Token(t) = &child {
let kind = crate::BeancountLanguage::kind_from_raw(t.kind());
let (start, end) = (offset, offset + len);
if line_start.is_none() {
line_start = Some(start);
}
if kind == K::NEWLINE {
if first_non_trivia == Some(K::STAR)
&& let Some(ls) = line_start
{
out.push(Spanned::new(
String::new(),
Span::new(ls + bom_offset as usize, end + bom_offset as usize),
));
}
line_start = None;
first_non_trivia = None;
} else if first_non_trivia.is_none() && !is_trivia_kind(kind) {
first_non_trivia = Some(kind);
}
}
offset += len;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{SyntaxKind, parse_structured};
use rustledger_core::Directive;
fn red_spans(root: &crate::SyntaxNode, bom: u32) -> Vec<(SyntaxKind, Span)> {
root.children()
.map(|n| {
let r = n.text_range();
let s = (u32::from(r.start()) + bom) as usize;
let e = (u32::from(r.end()) + bom) as usize;
(n.kind(), Span::new(s, e))
})
.collect()
}
#[test]
fn green_node_spans_match_red() {
let cases = [
"",
"2020-01-01 open Assets:Cash USD\n",
"2020-01-01 * \"p\" \"m\"\n Assets:Cash 5.00 USD\n Income:X\n",
"; leading comment\n\n2020-01-01 open A\n2020-01-02 close A\n",
"option \"title\" \"x\"\n2020-01-01 commodity USD\n",
"2020-01-01 price AAPL 5 USD\n2020-01-01 balance A 0 USD\n",
"\u{feff}2020-01-01 open A\n",
"2020-01-01 * \"é\" \"münts\"\n A 1 EUR\n B\n",
"garbage !! line\n2020-01-01 open A\n",
"2020-01-01 txn \"x\"\n A 10 AAPL {2.00 USD}\n B -20.00 USD\n",
];
for src in cases {
let (stripped, has_bom) = crate::bom::strip_leading(src);
let bom = if has_bom { 3 } else { 0 };
let root = parse_structured(stripped);
assert_eq!(
top_level_node_spans(&root, bom),
red_spans(&root, bom),
"green vs red node spans diverged for: {src:?}"
);
}
}
#[test]
fn green_txn_header_matches_red() {
let cases = [
"2020-01-01 * \"payee\" \"narr\"\n A 1 USD\n B\n",
"2020-01-01 ! \"only narration\"\n A 1 USD\n B\n",
"2020-01-01 txn \"n\"\n A 1 USD\n B\n",
"2020-01-01 # \"flagged\"\n A 1 USD\n B\n",
"2020-01-01 *\n A 1 USD\n B\n",
"2020-01-01 * \"p\" \"n\" #tag1 #tag2 ^link-a\n A 1 USD\n B\n",
"2020-01-01 * \"esc \\\"q\\\" tab\\there\"\n A 1 USD\n B\n",
"2020-01-01 * \"é payee\" \"münts\"\n A 1 EUR\n B\n",
];
for src in cases {
let root = parse_structured(src);
let green = root.green();
let mut offset = 0usize;
let mut txn_node = None;
for child in green.children() {
let len = match &child {
NodeOrToken::Node(n) => u32::from(n.text_len()) as usize,
NodeOrToken::Token(t) => u32::from(t.text_len()) as usize,
};
if let NodeOrToken::Node(n) = child
&& crate::BeancountLanguage::kind_from_raw(n.kind()) == SyntaxKind::TRANSACTION
{
txn_node = Some((n, offset));
break;
}
offset += len;
}
let (txn_green, base) = txn_node.expect("transaction node");
let (g, g_span) = convert_transaction_header(txn_green, base).expect("green header");
let red = crate::parse(src);
let red_sp = &red.directives[0];
let Directive::Transaction(r) = &red_sp.value else {
panic!("expected transaction for {src:?}");
};
assert_eq!(g.date, r.date, "date {src:?}");
assert_eq!(g.flag, r.flag, "flag {src:?}");
assert_eq!(g.payee, r.payee, "payee {src:?}");
assert_eq!(g.narration, r.narration, "narration {src:?}");
assert_eq!(g.tags, r.tags, "tags {src:?}");
assert_eq!(g.links, r.links, "links {src:?}");
assert_eq!(g_span, red_sp.span, "span {src:?}");
}
}
#[test]
fn green_simple_posting_matches_red() {
let cases = [
"2020-01-01 * \"p\"\n Assets:Cash 5.00 USD\n Income:X\n",
"2020-01-01 * \"p\"\n Assets:A -10 EUR\n Assets:B 10 EUR\n",
"2020-01-01 * \"p\"\n A 5 USD ; note here\n B\n",
"2020-01-01 * \"p\"\n A 1234.5678 USD\n B 0 USD\n",
"2020-01-01 * \"p\"\n Assets:Stock 10 AAPL {2.00 USD}\n Assets:Cash -20.00 USD\n",
"2020-01-01 * \"p\"\n Assets:Stock 10 AAPL {{20.00 USD}}\n Assets:Cash -20.00 USD\n",
"2020-01-01 * \"p\"\n A 10 AAPL {2.00 USD, 2021-06-01}\n B -20.00 USD\n",
"2020-01-01 * \"p\"\n A 10 AAPL {2.00 USD, \"lot-1\"}\n B -20.00 USD\n",
"2020-01-01 * \"p\"\n A -5 AAPL {1.50 # 8.00 USD}\n B 8.00 USD\n Income:G\n",
"2020-01-01 * \"p\"\n A 10 AAPL @ 3.00 USD\n B -30.00 USD\n",
"2020-01-01 * \"p\"\n A 10 AAPL @@ 25.00 USD\n B -25.00 USD\n",
"2020-01-01 * \"p\"\n A -5 AAPL {2.00 USD} @ 3.00 USD\n B 15.00 USD\n Income:G\n",
];
for src in cases {
let red = crate::parse(src);
let Directive::Transaction(rtxn) = &red.directives[0].value else {
panic!("txn {src:?}");
};
let root = parse_structured(src);
let green = root.green();
let mut off = 0usize;
let mut txn = None;
for child in green.children() {
let len = match &child {
NodeOrToken::Node(n) => u32::from(n.text_len()) as usize,
NodeOrToken::Token(t) => u32::from(t.text_len()) as usize,
};
if let NodeOrToken::Node(n) = child
&& crate::BeancountLanguage::kind_from_raw(n.kind()) == SyntaxKind::TRANSACTION
{
txn = Some((n, off));
break;
}
off += len;
}
let (txn_node, txn_base) = txn.expect("txn node");
let mut poff = txn_base;
let mut gi = 0usize;
for child in txn_node.children() {
let len = match &child {
NodeOrToken::Node(n) => u32::from(n.text_len()) as usize,
NodeOrToken::Token(t) => u32::from(t.text_len()) as usize,
};
if let NodeOrToken::Node(n) = child
&& crate::BeancountLanguage::kind_from_raw(n.kind()) == SyntaxKind::POSTING
{
let gp = convert_simple_posting(n, poff).expect("simple posting");
let rp = &rtxn.postings[gi];
assert_eq!(gp.value, rp.value, "posting {gi} value {src:?}");
assert_eq!(gp.span, rp.span, "posting {gi} span {src:?}");
gi += 1;
}
poff += len;
}
assert_eq!(gi, rtxn.postings.len(), "posting count {src:?}");
}
}
#[test]
fn parse_green_eq_red_corpus() {
let corpus = [
"",
"2020-01-01 * \"p\" \"n\"\n A 1 USD\n B\n",
"2020-01-01 ! \"x\" #t ^l\n A 10 AAPL {2 USD} @ 3 USD\n B -20 USD\n Income:G\n",
"2020-01-01 * \"p\"\n ! A 1 USD\n B\n", "2020-01-01 * \"p\"\n A 1 USD\n note: \"m\"\n B\n", "2020-01-01 * \"p\"\n meta: \"x\"\n A 1 USD\n B\n", "2020-01-01 * \"p\"\n ; a comment\n A 1 USD\n B\n", "2020-01-01 * \"p\"\n A 5 USD + 3 USD\n B\n", "2020-01-01 * \"p\"\n A 5 USD 3 USD\n B\n", "2020-01-01 * \"p\" | \"n\"\n A 1 USD\n B\n", "2020-13-99 * \"bad date\"\n A 1 USD\n B\n", "2020-99-99 * \"x\" 2021-01-01\n A 1 USD\n B\n",
"3333/33/3 X\n", "2020-01-01 *\n Aa:B 1 USD{,{*",
"7046/7/1D\n\tA:F{7#\u{06f6}>0",
"\u{feff}2020-01-01 * \"p\"\n A 1 USD\n B\n", "2020-01-01 * \"é\" \"münts\"\n Aaa 1 EUR\n B\n", "garbage\n2020-01-01 open A\n2020-01-01 * \"p\"\n A 1 USD\n B\n", "2020-01-01 open A\n2020-01-02 close A\noption \"x\" \"y\"\n", "2020-01-01 * \"p\"\n ! Assets:A 1 USD\n * Assets:B -1 USD\n",
"2020-01-01 * \"p\"\n A Assets:Letter 1 USD\n Assets:B -1 USD\n",
"2020-01-01 * \"p\"\n # Assets:A 1 USD\n Assets:B -1 USD\n",
"2020-01-01 * \"p\"\n Assets:A 1 USD\n str: \"hello\"\n int: 42\n neg: -7\n dec: 3.14\n amt: 5.00 USD\n dt: 2021-06-01\n acct: Assets:Other\n cur: EUR\n yes: TRUE\n no: FALSE\n tg: #atag\n lk: ^alink\n empty:\n Assets:B -1 USD\n",
"2020-01-01 * \"p\"\n meta1: \"x\"\n count: 3\n Assets:A 1 USD\n Assets:B -1 USD\n",
"2020-01-01 * \"p\"\n ! Assets:S 10 AAPL {2 USD} @ 3 USD\n lot: \"q1\"\n Assets:C 15 USD\n Income:G\n",
"; column-0 comment\n ; indented comment\n2020-01-01 open Assets:A USD\n",
"2020-01-01 * \"p\"\n Assets:Cash 5 USD\n Income:Salary -5 EUR\n", "!!garbage Assets:InError 5 BAD!!\n2020-01-01 open Assets:Real USD\n", "* Org Section Heading\n** Sub\n2020-01-01 open A\n", "2020-01-01 open A\n2020-01-01 open B\n2020-01-01 open C\n", " 2020-01-01 open Assets:A USD\n", "\t2020-01-01 close Assets:A\n", "2020-01-01 custom \"budget\" USD\n", "2020-01-01 custom \"b\" 10 USD \"ok\" NZD\n", "2020-01-01 * \"p\"\n unexpected junk here\n A 1 USD\n B\n", "2020-01-01 * \"p\" #tag1\n A 1 USD\n #bodytag\n B\n", "@@@ totally invalid line @@@\n2020-01-01 open A\n", "* Section\n** Indented Sub\n; c0 comment\n2020-01-01 open A\n", ];
for src in corpus {
let g = crate::parse(src);
let r = crate::cst::parse_red_only(src);
let dbg = |p: &crate::ParseResult| {
(
format!("{:?}", p.directives),
format!("{:?}", p.errors),
format!("{:?}", p.comments),
format!("{:?}", p.options),
format!("{:?}", p.account_occurrences),
format!("{:?}", p.currency_occurrences),
)
};
assert_eq!(
dbg(&g),
dbg(&r),
"green-wired parse diverged from red for: {src:?}"
);
}
}
}