use crate::base62::decode_base62;
use crate::error::{DxError, Result};
use crate::schema::{Schema, TypeHint};
use crate::tokenizer::{Token, Tokenizer};
use crate::types::{DxArray, DxObject, DxTable, DxValue};
use rustc_hash::FxHashMap;
pub struct Parser<'a> {
tokenizer: Tokenizer<'a>,
aliases: FxHashMap<String, String>,
anchors: Vec<DxValue>,
prefix_stack: Vec<String>,
schemas: FxHashMap<String, Schema>,
auto_counters: FxHashMap<String, i64>,
}
impl<'a> Parser<'a> {
pub fn new(input: &'a [u8]) -> Self {
Self {
tokenizer: Tokenizer::new(input),
aliases: FxHashMap::default(),
anchors: Vec::new(),
auto_counters: FxHashMap::default(),
prefix_stack: Vec::new(),
schemas: FxHashMap::default(),
}
}
pub fn parse(&mut self) -> Result<DxValue> {
let mut root = DxObject::new();
loop {
self.tokenizer.skip_whitespace();
if self.tokenizer.is_eof() {
break;
}
let token = self.tokenizer.peek_token()?;
match token {
Token::Eof => break,
Token::Newline => {
self.tokenizer.next_token()?;
continue;
}
Token::Dollar => {
if self.is_alias_definition()? {
self.parse_alias()?;
} else {
let (key, value) = self.parse_key_value_with_alias()?;
root.insert(key, value);
}
}
Token::Ident(_) | Token::Caret => {
let (key, value) = self.parse_key_value()?;
root.insert(key, value);
}
_ => {
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: format!("Unexpected token: {:?}", token),
});
}
}
}
Ok(DxValue::Object(root))
}
fn is_alias_definition(&mut self) -> Result<bool> {
let saved_pos = self.tokenizer.pos();
self.tokenizer.next_token()?;
let alias_token = self.tokenizer.next_token()?;
if !matches!(alias_token, Token::Ident(_)) {
self.tokenizer.reset_to(saved_pos);
return Ok(false);
}
self.tokenizer.skip_whitespace();
let next = self.tokenizer.peek_token()?;
let is_definition = matches!(next, Token::Equals);
self.tokenizer.reset_to(saved_pos);
Ok(is_definition)
}
fn parse_key_value_with_alias(&mut self) -> Result<(String, DxValue)> {
self.tokenizer.next_token()?;
let full_ident = match self.tokenizer.next_token()? {
Token::Ident(bytes) => std::str::from_utf8(bytes)?.to_string(),
_ => {
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: "Expected alias name after $".to_string(),
});
}
};
let (alias, suffix) = if let Some(dot_pos) = full_ident.find('.') {
(&full_ident[..dot_pos], Some(&full_ident[dot_pos..]))
} else {
(full_ident.as_str(), None)
};
let resolved = self
.aliases
.get(alias)
.cloned()
.ok_or_else(|| DxError::UnknownAlias(alias.to_string()))?;
let key = if let Some(suffix) = suffix {
format!("{}{}", resolved, suffix)
} else {
resolved
};
self.tokenizer.skip_whitespace();
let operator = self.tokenizer.next_token()?;
let value = match operator {
Token::Colon => self.parse_value()?,
Token::Bang => DxValue::Bool(true),
Token::Void => DxValue::Null,
_ => {
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: format!("Expected :, !, or ? after key, got {:?}", operator),
});
}
};
Ok((key, value))
}
fn parse_alias(&mut self) -> Result<()> {
self.tokenizer.next_token()?;
let alias = match self.tokenizer.next_token()? {
Token::Ident(bytes) => std::str::from_utf8(bytes)?.to_string(),
_ => {
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: "Expected alias name after $".to_string(),
});
}
};
if !matches!(self.tokenizer.next_token()?, Token::Equals) {
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: "Expected = after alias".to_string(),
});
}
let value = match self.tokenizer.next_token()? {
Token::Ident(bytes) => std::str::from_utf8(bytes)?.to_string(),
_ => {
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: "Expected value after alias =".to_string(),
});
}
};
self.aliases.insert(alias, value);
Ok(())
}
fn parse_key_value(&mut self) -> Result<(String, DxValue)> {
let mut key = String::new();
if matches!(self.tokenizer.peek_token()?, Token::Caret) {
self.tokenizer.next_token()?;
if let Some(prefix) = self.prefix_stack.last() {
key.push_str(prefix);
key.push('.');
}
}
if self.prefix_stack.len() > crate::error::MAX_RECURSION_DEPTH {
return Err(DxError::recursion_limit_exceeded(self.prefix_stack.len()));
}
match self.tokenizer.next_token()? {
Token::Ident(bytes) => {
let key_str = std::str::from_utf8(bytes)?;
if let Some(alias_key) = key_str.strip_prefix('$') {
if let Some(resolved) = self.aliases.get(alias_key) {
key.push_str(resolved);
} else {
return Err(DxError::UnknownAlias(alias_key.to_string()));
}
} else {
key.push_str(key_str);
}
}
_ => {
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: "Expected key".to_string(),
});
}
}
let full_key = key.clone();
self.tokenizer.skip_whitespace();
let operator = self.tokenizer.next_token()?;
let value = match operator {
Token::Colon => {
self.prefix_stack.push(full_key.clone());
let val = self.parse_value()?;
self.prefix_stack.pop();
val
}
Token::Equals => {
self.parse_table_definition(&key)?
}
Token::Stream => {
self.parse_stream_array()?
}
Token::Bang => {
DxValue::Bool(true)
}
Token::Void => {
DxValue::Null
}
_ => {
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: format!("Expected :, =, or > after key, got {:?}", operator),
});
}
};
Ok((key, value))
}
fn parse_value(&mut self) -> Result<DxValue> {
self.tokenizer.skip_whitespace();
let token = self.tokenizer.next_token()?;
match token {
Token::Eof => Err(DxError::UnexpectedEof(self.tokenizer.pos())),
Token::Newline => {
Err(DxError::UnexpectedEof(self.tokenizer.pos()))
}
Token::Null | Token::Void => Ok(DxValue::Null),
Token::True => Ok(DxValue::Bool(true)),
Token::False => Ok(DxValue::Bool(false)),
Token::Int(i) => Ok(DxValue::Int(i)),
Token::Float(f) => Ok(DxValue::Float(f)),
Token::Ditto => Err(DxError::DittoNoPrevious(self.tokenizer.pos())),
Token::At => {
let anchor_token = self.tokenizer.next_token()?;
let anchor_id = match anchor_token {
Token::Eof => {
return Err(DxError::UnexpectedEof(self.tokenizer.pos()));
}
Token::Int(i) => i as usize,
_ => {
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: "Expected number after @".to_string(),
});
}
};
self.anchors
.get(anchor_id)
.cloned()
.ok_or_else(|| DxError::UnknownAnchor(anchor_id.to_string()))
}
Token::Ident(bytes) => {
let s = std::str::from_utf8(bytes)?;
Ok(DxValue::String(s.to_string()))
}
_ => Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: format!("Unexpected token in value: {:?}", token),
}),
}
}
fn parse_stream_array(&mut self) -> Result<DxValue> {
let mut values = Vec::new();
loop {
self.tokenizer.skip_whitespace();
let token = self.tokenizer.peek_token()?;
if matches!(token, Token::Newline | Token::Eof) {
break;
}
let val = self.parse_value()?;
values.push(val);
self.tokenizer.skip_whitespace();
if matches!(self.tokenizer.peek_token()?, Token::Pipe) {
self.tokenizer.next_token()?; } else {
break;
}
}
Ok(DxValue::Array(DxArray::stream(values)))
}
fn parse_table_definition(&mut self, name: &str) -> Result<DxValue> {
self.tokenizer.skip_whitespace();
let schema_line = self.tokenizer.read_until(b'\n');
let schema_str = std::str::from_utf8(schema_line)?;
let schema = Schema::parse_definition(name.to_string(), schema_str)?;
self.schemas.insert(name.to_string(), schema.clone());
if matches!(self.tokenizer.peek(), Some(b'\n')) {
self.tokenizer.advance(1);
}
let mut table = DxTable::new(schema.clone());
let mut prev_row: Option<Vec<DxValue>> = None;
loop {
self.tokenizer.skip_whitespace();
let token = self.tokenizer.peek_token()?;
if matches!(token, Token::Eof) {
break;
}
if matches!(token, Token::Newline) {
self.tokenizer.next_token()?;
continue;
}
if matches!(token, Token::Ident(_)) {
let saved_pos = self.tokenizer.pos();
self.tokenizer.next_token()?; let next = self.tokenizer.peek_token()?;
self.tokenizer.reset_to(saved_pos);
if matches!(
next,
Token::Colon
| Token::Equals
| Token::Stream
| Token::Bang
| Token::Void
| Token::Caret
| Token::Dollar
) {
break;
}
}
let row = self.parse_table_row(&schema, prev_row.as_ref())?;
if row.is_empty() {
break;
}
prev_row = Some(row.clone());
table.add_row(row).map_err(DxError::SchemaError)?;
if table.rows.len() > crate::error::MAX_TABLE_ROWS {
return Err(DxError::table_too_large(table.rows.len()));
}
self.tokenizer.skip_whitespace();
if matches!(self.tokenizer.peek(), Some(b'\n')) {
self.tokenizer.advance(1);
}
}
Ok(DxValue::Table(table))
}
fn parse_table_row(
&mut self,
schema: &Schema,
prev_row: Option<&Vec<DxValue>>,
) -> Result<Vec<DxValue>> {
let mut row = Vec::with_capacity(schema.columns.len());
for (col_idx, column) in schema.columns.iter().enumerate() {
self.tokenizer.skip_whitespace();
if matches!(column.type_hint, TypeHint::AutoIncrement) {
let counter = self.auto_counters.entry(schema.name.clone()).or_insert(1);
row.push(DxValue::Int(*counter));
*counter += 1;
continue;
}
let token = self.tokenizer.peek_token()?;
if matches!(token, Token::Eof) {
if row.is_empty() {
return Ok(row);
}
return Err(DxError::UnexpectedEof(self.tokenizer.pos()));
}
if matches!(token, Token::Newline) {
if row.is_empty() {
return Ok(row);
}
return Err(DxError::InvalidSyntax {
pos: self.tokenizer.pos(),
msg: format!(
"Incomplete table row: expected {} columns, got {}",
schema.columns.len(),
row.len()
),
});
}
if matches!(token, Token::Ditto) {
self.tokenizer.next_token()?;
if let Some(prev) = prev_row {
if col_idx < prev.len() {
row.push(prev[col_idx].clone());
} else {
return Err(DxError::DittoNoPrevious(self.tokenizer.pos()));
}
} else {
return Err(DxError::DittoNoPrevious(self.tokenizer.pos()));
}
continue;
}
let value = match column.type_hint {
TypeHint::Int => match self.tokenizer.next_token()? {
Token::Eof => {
return Err(DxError::UnexpectedEof(self.tokenizer.pos()));
}
Token::Int(i) => DxValue::Int(i),
Token::Ditto => {
if let Some(prev) = prev_row {
if col_idx < prev.len() {
prev[col_idx].clone()
} else {
return Err(DxError::DittoNoPrevious(self.tokenizer.pos()));
}
} else {
return Err(DxError::DittoNoPrevious(self.tokenizer.pos()));
}
}
other => {
return Err(DxError::TypeMismatch {
expected: "int".to_string(),
actual: format!("{:?}", other),
});
}
},
TypeHint::Float => match self.tokenizer.next_token()? {
Token::Eof => {
return Err(DxError::UnexpectedEof(self.tokenizer.pos()));
}
Token::Float(f) => DxValue::Float(f),
Token::Int(i) => DxValue::Float(i as f64),
other => {
return Err(DxError::TypeMismatch {
expected: "float".to_string(),
actual: format!("{:?}", other),
});
}
},
TypeHint::Bool => match self.tokenizer.next_token()? {
Token::Eof => {
return Err(DxError::UnexpectedEof(self.tokenizer.pos()));
}
Token::True => DxValue::Bool(true),
Token::False => DxValue::Bool(false),
other => {
return Err(DxError::TypeMismatch {
expected: "bool".to_string(),
actual: format!("{:?}", other),
});
}
},
TypeHint::Base62 => {
match self.tokenizer.next_token()? {
Token::Eof => {
return Err(DxError::UnexpectedEof(self.tokenizer.pos()));
}
Token::Ident(bytes) => {
let s = std::str::from_utf8(bytes)?;
let n = decode_base62(s)?;
DxValue::Int(n as i64)
}
Token::Int(i) => DxValue::Int(i), other => {
return Err(DxError::TypeMismatch {
expected: "base62".to_string(),
actual: format!("{:?}", other),
});
}
}
}
TypeHint::String => {
let next_is_number = col_idx + 1 < schema.columns.len()
&& matches!(
schema.columns[col_idx + 1].type_hint,
TypeHint::Int | TypeHint::Float | TypeHint::Base62
);
let bytes = self.tokenizer.read_string_vacuum(next_is_number);
let s = std::str::from_utf8(bytes)?.trim().to_string();
DxValue::String(s)
}
TypeHint::AutoIncrement => {
unreachable!("AutoIncrement handled before loop")
}
TypeHint::Auto => self.parse_value()?,
};
row.push(value);
}
Ok(row)
}
}
#[must_use = "parsing result should be used"]
pub fn parse(input: &[u8]) -> Result<DxValue> {
if input.len() > crate::error::MAX_INPUT_SIZE {
return Err(DxError::input_too_large(input.len()));
}
let mut parser = Parser::new(input);
parser.parse()
}
pub fn parse_str(input: &str) -> Result<DxValue> {
parse(input.as_bytes())
}
pub fn parse_stream<R: std::io::Read>(reader: R) -> Result<DxValue> {
let mut buffer = Vec::new();
let mut reader = reader;
reader.read_to_end(&mut buffer)?;
parse(&buffer)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_simple_parse() {
let input = b"name:Alice
age:30
active:+";
let result = parse(input).unwrap();
if let DxValue::Object(obj) = result {
assert_eq!(obj.get("name"), Some(&DxValue::String("Alice".to_string())));
assert_eq!(obj.get("age"), Some(&DxValue::Int(30)));
assert_eq!(obj.get("active"), Some(&DxValue::Bool(true)));
} else {
panic!("Expected object");
}
}
#[test]
fn test_table_parse() {
let input = b"users=id%i name%s active%b
1 Alice +
2 Bob -";
let result = parse(input).unwrap();
if let DxValue::Object(obj) = result {
if let Some(DxValue::Table(table)) = obj.get("users") {
assert_eq!(table.row_count(), 2);
assert_eq!(table.rows[0][0], DxValue::Int(1));
assert_eq!(table.rows[0][1], DxValue::String("Alice".to_string()));
} else {
panic!("Expected table");
}
}
}
#[test]
fn test_stream_array() {
let input = b"tags>alpha|beta|gamma";
let result = parse(input).unwrap();
if let DxValue::Object(obj) = result {
if let Some(DxValue::Array(arr)) = obj.get("tags") {
assert!(arr.is_stream);
assert_eq!(arr.values.len(), 3);
} else {
panic!("Expected array");
}
}
}
#[test]
fn test_alias() {
let input = b"$c=context
$c.task:Mission";
let result = parse(input).unwrap();
if let DxValue::Object(obj) = result {
assert_eq!(
obj.get("context.task"),
Some(&DxValue::String("Mission".to_string()))
);
}
}
#[test]
fn test_ditto() {
let input = b"data=id%i name%s
1 Alice
_ Bob";
let result = parse(input).unwrap();
if let DxValue::Object(obj) = result {
if let Some(DxValue::Table(table)) = obj.get("data") {
assert_eq!(table.rows[1][0], DxValue::Int(1)); }
}
}
#[test]
fn test_eof_in_value() {
let input = b"key:";
let result = parse(input);
assert!(result.is_err(), "EOF after colon should error");
if let Err(DxError::UnexpectedEof(pos)) = result {
assert!(pos <= input.len(), "Position should be within input bounds");
}
}
#[test]
fn test_eof_in_anchor_reference() {
let input = b"key:@";
let result = parse(input);
assert!(result.is_err(), "EOF after @ should error");
}
#[test]
fn test_empty_input_parses() {
let result = parse(b"");
assert!(result.is_ok(), "Empty input should parse successfully");
if let Ok(DxValue::Object(obj)) = result {
assert!(
obj.fields.is_empty(),
"Empty input should produce empty object"
);
}
}
fn _assert_dx_value_send<T: Send>() {}
fn _assert_dx_value_sync<T: Sync>() {}
#[test]
fn test_dx_value_is_send_sync() {
_assert_dx_value_send::<DxValue>();
_assert_dx_value_sync::<DxValue>();
}
#[test]
fn test_parser_is_send() {
fn assert_send<T: Send>() {}
assert_send::<Parser<'_>>();
}
#[test]
fn test_parse_is_stateless() {
use std::sync::Arc;
use std::thread;
let input = Arc::new(b"name:Test\nvalue:42".to_vec());
let num_threads = 4;
let handles: Vec<_> = (0..num_threads)
.map(|_| {
let input_clone = Arc::clone(&input);
thread::spawn(move || parse(&input_clone))
})
.collect();
let results: Vec<_> = handles
.into_iter()
.map(|h| h.join().expect("Thread panicked"))
.collect();
let first = results[0].as_ref().expect("First parse failed");
for (i, result) in results.iter().enumerate().skip(1) {
let value = result
.as_ref()
.unwrap_or_else(|_| panic!("Parse {} failed", i));
assert_eq!(first, value, "Thread {} produced different result", i);
}
}
}