Skip to main content

toon_format/decode/
parser.rs

1use serde_json::{
2    Map,
3    Number,
4    Value,
5};
6
7use crate::{
8    constants::{
9        KEYWORDS,
10        MAX_DEPTH,
11        QUOTED_KEY_MARKER,
12    },
13    decode::{
14        scanner::{
15            Scanner,
16            Token,
17        },
18        validation,
19    },
20    types::{
21        DecodeOptions,
22        Delimiter,
23        ErrorContext,
24        ToonError,
25        ToonResult,
26    },
27    utils::validation::validate_depth,
28};
29#[cfg(feature = "layout")]
30use crate::{
31    decode::layout_builder::LayoutBuilder,
32    layout::{
33        FieldDescriptor,
34        Layout,
35        NodeLayout,
36    },
37};
38
39/// Context for parsing arrays to determine correct indentation depth.
40///
41/// Arrays as the first field of list-item objects require special indentation:
42/// their content (rows for tabular, items for non-uniform) appears at depth +2
43/// relative to the hyphen line, while arrays in other contexts use depth +1.
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45enum ArrayParseContext {
46    /// Normal array parsing context (content at depth +1)
47    Normal,
48
49    /// Array as first field of list-item object
50    /// (content at depth +2 relative to hyphen line)
51    ListItemFirstField,
52}
53
54/// Parser that builds JSON values from a sequence of tokens.
55#[allow(unused)]
56pub struct Parser<'a> {
57    scanner: Scanner,
58    current_token: Token,
59    options: DecodeOptions,
60    delimiter: Option<Delimiter>,
61    input: &'a str,
62    #[cfg(feature = "layout")]
63    layout: Option<LayoutBuilder>,
64}
65
66impl<'a> Parser<'a> {
67    /// Create a new parser with the given input and options.
68    pub fn new(input: &'a str, options: DecodeOptions) -> ToonResult<Self> {
69        let mut scanner = Scanner::new(input);
70        let chosen_delim = options.delimiter;
71        scanner.set_active_delimiter(chosen_delim);
72        let current_token = scanner.scan_token()?;
73
74        Ok(Self {
75            scanner,
76            current_token,
77            delimiter: chosen_delim,
78            options,
79            input,
80            #[cfg(feature = "layout")]
81            layout: None,
82        })
83    }
84
85    #[cfg(feature = "layout")]
86    pub fn with_layout(mut self) -> Self {
87        self.layout = Some(LayoutBuilder::new());
88        self
89    }
90
91    #[cfg(feature = "layout")]
92    pub fn take_layout(&mut self) -> Option<Layout> {
93        self.layout.take().map(LayoutBuilder::finish)
94    }
95
96    fn layout_push(&mut self, segment: &str) {
97        #[cfg(feature = "layout")]
98        if let Some(b) = self.layout.as_mut() {
99            b.push(segment.to_string());
100        }
101        #[cfg(not(feature = "layout"))]
102        let _ = segment;
103    }
104
105    fn layout_pop(&mut self) {
106        #[cfg(feature = "layout")]
107        if let Some(b) = self.layout.as_mut() {
108            b.pop();
109        }
110    }
111
112    fn layout_record_tabular(&mut self, length: usize, fields: &[String], delimiter: Delimiter) {
113        #[cfg(feature = "layout")]
114        if let Some(b) = self.layout.as_mut() {
115            let descriptors: Vec<FieldDescriptor> =
116                fields.iter().map(FieldDescriptor::leaf).collect();
117            b.record(NodeLayout::Tabular {
118                declared_len: length,
119                fields: descriptors,
120                delimiter,
121            });
122        }
123        #[cfg(not(feature = "layout"))]
124        let _ = (length, fields, delimiter);
125    }
126
127    fn layout_record_list(&mut self, length: usize) {
128        #[cfg(feature = "layout")]
129        if let Some(b) = self.layout.as_mut() {
130            b.record(NodeLayout::List {
131                declared_len: length,
132            });
133        }
134        #[cfg(not(feature = "layout"))]
135        let _ = length;
136    }
137
138    fn layout_record_inline_array(&mut self, length: usize, delimiter: Delimiter) {
139        #[cfg(feature = "layout")]
140        if let Some(b) = self.layout.as_mut() {
141            b.record(NodeLayout::InlineArray {
142                declared_len: length,
143                delimiter,
144            });
145        }
146        #[cfg(not(feature = "layout"))]
147        let _ = (length, delimiter);
148    }
149
150    /// Parse the input into a JSON value.
151    pub fn parse(&mut self) -> ToonResult<Value> {
152        if self.options.strict {
153            self.validate_indentation(self.scanner.get_last_line_indent())?;
154        }
155        let value = self.parse_value()?;
156
157        // In strict mode, check for trailing content at root level
158        if self.options.strict {
159            self.skip_newlines()?;
160            if !matches!(self.current_token, Token::Eof) {
161                return Err(self
162                    .parse_error_with_context(
163                        "Multiple values at root level are not allowed in strict mode",
164                    )
165                    .with_suggestion("Wrap multiple values in an object or array"));
166            }
167        }
168
169        Ok(value)
170    }
171
172    fn advance(&mut self) -> ToonResult<()> {
173        self.current_token = self.scanner.scan_token()?;
174        Ok(())
175    }
176
177    fn skip_newlines(&mut self) -> ToonResult<()> {
178        while matches!(self.current_token, Token::Newline) {
179            self.advance()?;
180        }
181        Ok(())
182    }
183
184    fn parse_value(&mut self) -> ToonResult<Value> {
185        self.parse_value_with_depth(0)
186    }
187
188    fn parse_value_with_depth(&mut self, depth: usize) -> ToonResult<Value> {
189        validate_depth(depth, MAX_DEPTH)?;
190
191        let had_newline = matches!(self.current_token, Token::Newline);
192        self.skip_newlines()?;
193
194        match &self.current_token {
195            Token::Null => {
196                // Peek ahead to see if this is a key (followed by ':') or a value
197                let next_char_is_colon = matches!(self.scanner.peek(), Some(':'));
198                if next_char_is_colon {
199                    let key = KEYWORDS[0].to_string();
200                    self.advance()?;
201                    self.parse_object_with_initial_key(key, depth)
202                } else {
203                    self.advance()?;
204                    Ok(Value::Null)
205                }
206            }
207            Token::Bool(b) => {
208                let next_char_is_colon = matches!(self.scanner.peek(), Some(':'));
209                if next_char_is_colon {
210                    let key = if *b {
211                        KEYWORDS[1].to_string()
212                    } else {
213                        KEYWORDS[2].to_string()
214                    };
215                    self.advance()?;
216                    self.parse_object_with_initial_key(key, depth)
217                } else {
218                    let val = *b;
219                    self.advance()?;
220                    Ok(Value::Bool(val))
221                }
222            }
223            Token::SignedInteger(i) => {
224                let next_char_is_colon = matches!(self.scanner.peek(), Some(':'));
225                if next_char_is_colon {
226                    let key = i.to_string();
227                    self.advance()?;
228                    self.parse_object_with_initial_key(key, depth)
229                } else {
230                    let first_text = self.scanner.last_token_text().to_string();
231                    let val = *i;
232                    self.advance()?;
233                    // Check if followed by more value tokens on the same line
234                    match &self.current_token {
235                        Token::String(..)
236                        | Token::SignedInteger(..)
237                        | Token::UnsignedInteger(..)
238                        | Token::Number(..)
239                        | Token::Bool(..)
240                        | Token::Null => {
241                            let mut accumulated = first_text;
242                            while let Token::String(..)
243                            | Token::SignedInteger(..)
244                            | Token::UnsignedInteger(..)
245                            | Token::Number(..)
246                            | Token::Bool(..)
247                            | Token::Null = &self.current_token
248                            {
249                                let ws = self.scanner.last_whitespace_count().max(1);
250                                for _ in 0..ws {
251                                    accumulated.push(' ');
252                                }
253                                accumulated.push_str(self.scanner.last_token_text());
254                                self.advance()?;
255                            }
256                            Ok(Value::String(accumulated))
257                        }
258                        _ => Ok(serde_json::Number::from(val).into()),
259                    }
260                }
261            }
262            Token::UnsignedInteger(i) => {
263                let next_char_is_colon = matches!(self.scanner.peek(), Some(':'));
264                if next_char_is_colon {
265                    let key = i.to_string();
266                    self.advance()?;
267                    self.parse_object_with_initial_key(key, depth)
268                } else {
269                    let first_text = self.scanner.last_token_text().to_string();
270                    let val = *i;
271                    self.advance()?;
272                    // Check if followed by more value tokens on the same line
273                    match &self.current_token {
274                        Token::String(..)
275                        | Token::SignedInteger(..)
276                        | Token::UnsignedInteger(..)
277                        | Token::Number(..)
278                        | Token::Bool(..)
279                        | Token::Null => {
280                            let mut accumulated = first_text;
281                            while let Token::String(..)
282                            | Token::SignedInteger(..)
283                            | Token::UnsignedInteger(..)
284                            | Token::Number(..)
285                            | Token::Bool(..)
286                            | Token::Null = &self.current_token
287                            {
288                                let ws = self.scanner.last_whitespace_count().max(1);
289                                for _ in 0..ws {
290                                    accumulated.push(' ');
291                                }
292                                accumulated.push_str(self.scanner.last_token_text());
293                                self.advance()?;
294                            }
295                            Ok(Value::String(accumulated))
296                        }
297                        _ => Ok(serde_json::Number::from(val).into()),
298                    }
299                }
300            }
301            Token::Number(n) => {
302                let next_char_is_colon = matches!(self.scanner.peek(), Some(':'));
303                if next_char_is_colon {
304                    let key = n.to_string();
305                    self.advance()?;
306                    self.parse_object_with_initial_key(key, depth)
307                } else {
308                    let first_text = self.scanner.last_token_text().to_string();
309                    let val = *n;
310                    self.advance()?;
311                    // Check if followed by more value tokens on the same line
312                    match &self.current_token {
313                        Token::String(..)
314                        | Token::SignedInteger(..)
315                        | Token::UnsignedInteger(..)
316                        | Token::Number(..)
317                        | Token::Bool(..)
318                        | Token::Null => {
319                            let mut accumulated = first_text;
320                            while let Token::String(..)
321                            | Token::SignedInteger(..)
322                            | Token::UnsignedInteger(..)
323                            | Token::Number(..)
324                            | Token::Bool(..)
325                            | Token::Null = &self.current_token
326                            {
327                                let ws = self.scanner.last_whitespace_count().max(1);
328                                for _ in 0..ws {
329                                    accumulated.push(' ');
330                                }
331                                accumulated.push_str(self.scanner.last_token_text());
332                                self.advance()?;
333                            }
334                            Ok(Value::String(accumulated))
335                        }
336                        _ => {
337                            // Normalize floats that are actually integers
338                            if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64
339                            {
340                                Ok(serde_json::Number::from(val as i64).into())
341                            } else {
342                                Ok(serde_json::Number::from_f64(val)
343                                    .ok_or_else(|| {
344                                        ToonError::InvalidInput(format!("Invalid number: {val}"))
345                                    })?
346                                    .into())
347                            }
348                        }
349                    }
350                }
351            }
352            Token::String(s, _) => {
353                let first = s.clone();
354                self.advance()?;
355
356                match &self.current_token {
357                    Token::Colon | Token::LeftBracket => {
358                        self.parse_object_with_initial_key(first, depth)
359                    }
360                    _ => {
361                        // Strings on new indented lines could be missing colons (keys) or values
362                        // Only error in strict mode when we know it's a new line
363                        if self.options.strict && depth > 0 && had_newline {
364                            return Err(self
365                                .parse_error_with_context(format!(
366                                    "Expected ':' after '{first}' in object context"
367                                ))
368                                .with_suggestion(
369                                    "Add ':' after the key, or place the value on the same line \
370                                     as the parent key",
371                                ));
372                        }
373
374                        if matches!(self.current_token, Token::Newline | Token::Eof) {
375                            return Ok(Value::String(first));
376                        }
377                        // Root-level string value - join consecutive tokens with exact spacing
378                        let mut accumulated = first;
379                        while let Token::String(..)
380                        | Token::SignedInteger(..)
381                        | Token::UnsignedInteger(..)
382                        | Token::Number(..)
383                        | Token::Bool(..)
384                        | Token::Null = &self.current_token
385                        {
386                            let ws = self.scanner.last_whitespace_count().max(1);
387                            for _ in 0..ws {
388                                accumulated.push(' ');
389                            }
390                            accumulated.push_str(self.scanner.last_token_text());
391                            self.advance()?;
392                        }
393                        Ok(Value::String(accumulated))
394                    }
395                }
396            }
397            Token::LeftBracket => self.parse_root_array(depth),
398            Token::Eof => Ok(Value::Object(Map::new())),
399            _ => self.parse_object(depth),
400        }
401    }
402
403    fn parse_object(&mut self, depth: usize) -> ToonResult<Value> {
404        validate_depth(depth, MAX_DEPTH)?;
405
406        let mut obj = Map::new();
407        // Track the indentation of the first key to ensure all keys align
408        let mut base_indent: Option<usize> = None;
409
410        loop {
411            while matches!(self.current_token, Token::Newline) {
412                self.advance()?;
413            }
414
415            if matches!(self.current_token, Token::Eof) {
416                break;
417            }
418
419            let current_indent = self.scanner.get_last_line_indent();
420
421            if self.options.strict {
422                self.validate_indentation(current_indent)?;
423            }
424
425            // Once we've seen the first key, all subsequent keys must match its indent
426            if let Some(expected) = base_indent {
427                if current_indent != expected {
428                    break;
429                }
430            } else {
431                base_indent = Some(current_indent);
432            }
433
434            let key = match &self.current_token {
435                Token::String(s, was_quoted) => {
436                    // Mark quoted keys containing dots with a special prefix
437                    // so path expansion can skip them
438                    if *was_quoted && s.contains('.') {
439                        format!("{QUOTED_KEY_MARKER}{s}")
440                    } else {
441                        s.clone()
442                    }
443                }
444                _ => {
445                    return Err(self
446                        .parse_error_with_context(format!(
447                            "Expected key, found {:?}",
448                            self.current_token
449                        ))
450                        .with_suggestion("Object keys must be strings"));
451                }
452            };
453            self.advance()?;
454
455            self.layout_push(&key);
456            let value = if matches!(self.current_token, Token::LeftBracket) {
457                self.parse_array(depth)?
458            } else {
459                if !matches!(self.current_token, Token::Colon) {
460                    return Err(self
461                        .parse_error_with_context(format!(
462                            "Expected ':' or '[', found {:?}",
463                            self.current_token
464                        ))
465                        .with_suggestion("Use ':' for object values or '[' for arrays"));
466                }
467                self.advance()?;
468                self.parse_field_value(depth)?
469            };
470            self.layout_pop();
471
472            obj.insert(key, value);
473        }
474
475        Ok(Value::Object(obj))
476    }
477
478    fn parse_object_with_initial_key(&mut self, key: String, depth: usize) -> ToonResult<Value> {
479        validate_depth(depth, MAX_DEPTH)?;
480
481        let mut obj = Map::new();
482        let mut base_indent: Option<usize> = None;
483
484        // Validate indentation for the initial key if in strict mode
485        if self.options.strict {
486            let current_indent = self.scanner.get_last_line_indent();
487            self.validate_indentation(current_indent)?;
488        }
489
490        self.layout_push(&key);
491        if matches!(self.current_token, Token::LeftBracket) {
492            let value = self.parse_array(depth)?;
493            self.layout_pop();
494            obj.insert(key, value);
495        } else {
496            if !matches!(self.current_token, Token::Colon) {
497                return Err(self.parse_error_with_context(format!(
498                    "Expected ':', found {:?}",
499                    self.current_token
500                )));
501            }
502            self.advance()?;
503
504            let value = self.parse_field_value(depth)?;
505            self.layout_pop();
506            obj.insert(key, value);
507        }
508
509        loop {
510            // Skip newlines and check if the next line belongs to this object
511            while matches!(self.current_token, Token::Newline) {
512                self.advance()?;
513
514                if !self.options.strict {
515                    while matches!(self.current_token, Token::Newline) {
516                        self.advance()?;
517                    }
518                }
519
520                if matches!(self.current_token, Token::Newline) {
521                    continue;
522                }
523
524                let next_indent = self.scanner.get_last_line_indent();
525
526                // Check if the next line is at the right indentation level
527                let should_continue = if let Some(expected) = base_indent {
528                    next_indent == expected
529                } else {
530                    // First field: use depth-based expected indent
531                    let current_depth_indent = self.options.indent.get_spaces() * depth;
532                    next_indent == current_depth_indent
533                };
534
535                if !should_continue {
536                    break;
537                }
538            }
539
540            if matches!(self.current_token, Token::Eof) {
541                break;
542            }
543
544            if !matches!(self.current_token, Token::String(_, _)) {
545                break;
546            }
547
548            if matches!(self.current_token, Token::Eof) {
549                break;
550            }
551
552            let current_indent = self.scanner.get_last_line_indent();
553
554            if let Some(expected) = base_indent {
555                if current_indent != expected {
556                    break;
557                }
558            } else {
559                // verify first additional field matches expected depth
560                let expected_depth_indent = self.options.indent.get_spaces() * depth;
561                if current_indent != expected_depth_indent {
562                    break;
563                }
564            }
565
566            if self.options.strict {
567                self.validate_indentation(current_indent)?;
568            }
569
570            if base_indent.is_none() {
571                base_indent = Some(current_indent);
572            }
573
574            let key = match &self.current_token {
575                Token::String(s, was_quoted) => {
576                    // Mark quoted keys containing dots with a special prefix
577                    // so path expansion can skip them
578                    if *was_quoted && s.contains('.') {
579                        format!("{QUOTED_KEY_MARKER}{s}")
580                    } else {
581                        s.clone()
582                    }
583                }
584                _ => break,
585            };
586            self.advance()?;
587
588            self.layout_push(&key);
589            let value = if matches!(self.current_token, Token::LeftBracket) {
590                self.parse_array(depth)?
591            } else {
592                if !matches!(self.current_token, Token::Colon) {
593                    self.layout_pop();
594                    break;
595                }
596                self.advance()?;
597                self.parse_field_value(depth)?
598            };
599            self.layout_pop();
600
601            obj.insert(key, value);
602        }
603
604        Ok(Value::Object(obj))
605    }
606
607    fn parse_field_value(&mut self, depth: usize) -> ToonResult<Value> {
608        validate_depth(depth, MAX_DEPTH)?;
609
610        if matches!(self.current_token, Token::Newline | Token::Eof) {
611            let has_children = if matches!(self.current_token, Token::Newline) {
612                let current_depth_indent = self.options.indent.get_spaces() * (depth + 1);
613                let next_indent = self.scanner.count_leading_spaces();
614                next_indent >= current_depth_indent
615            } else {
616                false
617            };
618
619            if has_children {
620                self.parse_value_with_depth(depth + 1)
621            } else {
622                Ok(Value::Object(Map::new()))
623            }
624        } else if matches!(self.current_token, Token::LeftBracket) {
625            self.parse_value_with_depth(depth + 1)
626        } else {
627            // Check if there's more content after the current token
628            let token_text = self.scanner.last_token_text().to_string();
629            let (rest, space_count) = self.scanner.read_rest_of_line_with_space_info();
630
631            let result = if rest.is_empty() && space_count == 0 {
632                // Single token - convert directly to avoid redundant parsing
633                match &self.current_token {
634                    Token::String(s, _) => Ok(Value::String(s.clone())),
635                    Token::SignedInteger(i) => Ok(serde_json::Number::from(*i).into()),
636                    Token::UnsignedInteger(i) => Ok(serde_json::Number::from(*i).into()),
637                    Token::Number(n) => {
638                        let val = *n;
639                        if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64 {
640                            Ok(serde_json::Number::from(val as i64).into())
641                        } else {
642                            Ok(serde_json::Number::from_f64(val)
643                                .ok_or_else(|| {
644                                    ToonError::InvalidInput(format!("Invalid number: {val}"))
645                                })?
646                                .into())
647                        }
648                    }
649                    Token::Bool(b) => Ok(Value::Bool(*b)),
650                    Token::Null => Ok(Value::Null),
651                    _ => Err(self.parse_error_with_context("Unexpected token after colon")),
652                }
653            } else {
654                // Multi-token value - reconstruct using original token text and re-parse
655                let mut value_str = match &self.current_token {
656                    Token::String(_, true) => {
657                        // Quoted strings: use last_token_text which includes quotes
658                        token_text.clone()
659                    }
660                    Token::String(_, false)
661                    | Token::SignedInteger(_)
662                    | Token::UnsignedInteger(_)
663                    | Token::Number(_)
664                    | Token::Bool(_)
665                    | Token::Null => token_text.clone(),
666                    _ => {
667                        return Err(self.parse_error_with_context("Unexpected token after colon"));
668                    }
669                };
670
671                // Preserve exact spacing from the original input
672                for _ in 0..space_count {
673                    value_str.push(' ');
674                }
675                value_str.push_str(&rest);
676
677                let token = self.scanner.parse_value_string(&value_str)?;
678                match token {
679                    Token::String(s, _) => Ok(Value::String(s)),
680                    Token::SignedInteger(i) => Ok(serde_json::Number::from(i).into()),
681                    Token::UnsignedInteger(i) => Ok(serde_json::Number::from(i).into()),
682                    Token::Number(n) => {
683                        if n.is_finite() && n.fract() == 0.0 && n.abs() <= i64::MAX as f64 {
684                            Ok(serde_json::Number::from(n as i64).into())
685                        } else {
686                            Ok(serde_json::Number::from_f64(n)
687                                .ok_or_else(|| {
688                                    ToonError::InvalidInput(format!("Invalid number: {n}"))
689                                })?
690                                .into())
691                        }
692                    }
693                    Token::Bool(b) => Ok(Value::Bool(b)),
694                    Token::Null => Ok(Value::Null),
695                    _ => Err(ToonError::InvalidInput("Unexpected token type".to_string())),
696                }
697            }?;
698
699            self.current_token = self.scanner.scan_token()?;
700            Ok(result)
701        }
702    }
703
704    fn parse_root_array(&mut self, depth: usize) -> ToonResult<Value> {
705        validate_depth(depth, MAX_DEPTH)?;
706
707        if !matches!(self.current_token, Token::LeftBracket) {
708            return Err(self.parse_error_with_context("Expected '[' at the start of root array"));
709        }
710
711        self.parse_array(depth)
712    }
713
714    fn parse_array_header(
715        &mut self,
716    ) -> ToonResult<(usize, Option<Delimiter>, Option<Vec<String>>)> {
717        if !matches!(self.current_token, Token::LeftBracket) {
718            return Err(self.parse_error_with_context("Expected '['"));
719        }
720        self.advance()?;
721
722        // Parse array length (plain integer only)
723        // Supports formats: [N], [N|], [N\t] (no # marker)
724        let length = if let Token::SignedInteger(n) = &self.current_token {
725            *n as usize
726        } else if let Token::UnsignedInteger(n) = &self.current_token {
727            *n as usize
728        } else if let Token::String(s, _) = &self.current_token {
729            // Check if string starts with # - this marker is not supported
730            if s.starts_with('#') {
731                return Err(self
732                    .parse_error_with_context(
733                        "Length marker '#' is not supported. Use [N] format instead of [#N]",
734                    )
735                    .with_suggestion("Remove the '#' prefix from the array length"));
736            }
737
738            // Plain string that's a number: "3"
739            s.parse::<usize>().map_err(|_| {
740                self.parse_error_with_context(format!("Expected array length, found: {s}"))
741            })?
742        } else {
743            return Err(self.parse_error_with_context(format!(
744                "Expected array length, found {:?}",
745                self.current_token
746            )));
747        };
748
749        self.advance()?;
750
751        // Check for optional delimiter after length
752        let detected_delim = match &self.current_token {
753            Token::Delimiter(d) => {
754                let delim = *d;
755                self.advance()?;
756                Some(delim)
757            }
758            Token::String(s, _) if s == "," => {
759                self.advance()?;
760                Some(Delimiter::Comma)
761            }
762            Token::String(s, _) if s == "|" => {
763                self.advance()?;
764                Some(Delimiter::Pipe)
765            }
766            Token::String(s, _) if s == "\t" => {
767                self.advance()?;
768                Some(Delimiter::Tab)
769            }
770            _ => None,
771        };
772
773        // Default to comma if no delimiter specified
774        let active_delim = detected_delim.or(Some(Delimiter::Comma));
775
776        self.scanner.set_active_delimiter(active_delim);
777
778        if !matches!(self.current_token, Token::RightBracket) {
779            return Err(self.parse_error_with_context(format!(
780                "Expected ']', found {:?}",
781                self.current_token
782            )));
783        }
784        self.advance()?;
785
786        let fields = if matches!(self.current_token, Token::LeftBrace) {
787            self.advance()?;
788            let mut fields = Vec::new();
789
790            loop {
791                match &self.current_token {
792                    Token::String(s, _) => {
793                        fields.push(s.clone());
794                        self.advance()?;
795
796                        if matches!(self.current_token, Token::RightBrace) {
797                            break;
798                        }
799
800                        if matches!(self.current_token, Token::Delimiter(_)) {
801                            self.advance()?;
802                        } else {
803                            return Err(self.parse_error_with_context(format!(
804                                "Expected delimiter or '}}', found {:?}",
805                                self.current_token
806                            )));
807                        }
808                    }
809                    Token::RightBrace => break,
810                    _ => {
811                        return Err(self.parse_error_with_context(format!(
812                            "Expected field name, found {:?}",
813                            self.current_token
814                        )))
815                    }
816                }
817            }
818
819            self.advance()?;
820            Some(fields)
821        } else {
822            None
823        };
824
825        if !matches!(self.current_token, Token::Colon) {
826            return Err(self.parse_error_with_context("Expected ':' after array header"));
827        }
828        self.advance()?;
829
830        Ok((length, detected_delim, fields))
831    }
832
833    fn parse_array(&mut self, depth: usize) -> ToonResult<Value> {
834        self.parse_array_with_context(depth, ArrayParseContext::Normal)
835    }
836
837    fn parse_array_with_context(
838        &mut self,
839        depth: usize,
840        context: ArrayParseContext,
841    ) -> ToonResult<Value> {
842        validate_depth(depth, MAX_DEPTH)?;
843
844        let (length, detected_delim, fields) = self.parse_array_header()?;
845        let delim = detected_delim.unwrap_or(Delimiter::Comma);
846
847        if let Some(fields) = fields {
848            validation::validate_field_list(&fields)?;
849            self.layout_record_tabular(length, &fields, delim);
850            self.parse_tabular_array(length, &fields, depth, context)
851        } else {
852            // Non-tabular arrays as first field of list items require depth adjustment
853            // (items at depth +2 relative to hyphen, not the usual +1)
854            let adjusted_depth = match context {
855                ArrayParseContext::Normal => depth,
856                ArrayParseContext::ListItemFirstField => depth + 1,
857            };
858            if length == 0 || matches!(self.current_token, Token::Newline) {
859                self.layout_record_list(length);
860            } else {
861                self.layout_record_inline_array(length, delim);
862            }
863            self.parse_regular_array(length, adjusted_depth)
864        }
865    }
866
867    fn parse_tabular_array(
868        &mut self,
869        length: usize,
870        fields: &[String],
871        depth: usize,
872        context: ArrayParseContext,
873    ) -> ToonResult<Value> {
874        let mut rows = Vec::new();
875
876        if !matches!(self.current_token, Token::Newline) {
877            return Err(self
878                .parse_error_with_context("Expected newline after tabular array header")
879                .with_suggestion("Tabular arrays must have rows on separate lines"));
880        }
881        self.skip_newlines()?;
882
883        for row_index in 0..length {
884            if matches!(self.current_token, Token::Eof) {
885                if self.options.strict {
886                    return Err(self.parse_error_with_context(format!(
887                        "Expected {} rows, but got {} before EOF",
888                        length,
889                        rows.len()
890                    )));
891                }
892                break;
893            }
894
895            let current_indent = self.scanner.get_last_line_indent();
896
897            // Tabular arrays as first field of list-item objects require rows at depth +2
898            // (relative to hyphen), while normal tabular arrays use depth +1
899            let row_depth_offset = match context {
900                ArrayParseContext::Normal => 1,
901                ArrayParseContext::ListItemFirstField => 2,
902            };
903            let expected_indent = self.options.indent.get_spaces() * (depth + row_depth_offset);
904
905            if self.options.strict {
906                self.validate_indentation(current_indent)?;
907
908                if current_indent != expected_indent {
909                    return Err(self.parse_error_with_context(format!(
910                        "Invalid indentation for tabular row: expected {expected_indent} spaces, \
911                         found {current_indent}"
912                    )));
913                }
914            }
915
916            let mut row = Map::new();
917
918            for (field_index, field) in fields.iter().enumerate() {
919                // Skip delimiter before each field except the first
920                if field_index > 0 {
921                    if matches!(self.current_token, Token::Delimiter(_)) {
922                        self.advance()?;
923                    } else {
924                        return Err(self
925                            .parse_error_with_context(format!(
926                                "Expected delimiter, found {:?}",
927                                self.current_token
928                            ))
929                            .with_suggestion(format!(
930                                "Tabular row {} field {} needs a delimiter",
931                                row_index + 1,
932                                field_index + 1
933                            )));
934                    }
935                }
936
937                // Empty values show up as delimiters or newlines
938                let value = if matches!(self.current_token, Token::Delimiter(_))
939                    || matches!(self.current_token, Token::Newline | Token::Eof)
940                {
941                    Value::String(String::new())
942                } else {
943                    self.parse_tabular_field_value()?
944                };
945
946                row.insert(field.clone(), value);
947
948                // Validate row completeness
949                if field_index < fields.len() - 1 {
950                    // Not the last field - shouldn't hit newline yet
951                    if matches!(self.current_token, Token::Newline | Token::Eof) {
952                        if self.options.strict {
953                            return Err(self
954                                .parse_error_with_context(format!(
955                                    "Tabular row {}: expected {} values, but found only {}",
956                                    row_index + 1,
957                                    fields.len(),
958                                    field_index + 1
959                                ))
960                                .with_suggestion(format!(
961                                    "Row {} should have exactly {} values",
962                                    row_index + 1,
963                                    fields.len()
964                                )));
965                        } else {
966                            // Fill remaining fields with null in non-strict mode
967                            for field in fields.iter().skip(field_index + 1) {
968                                row.insert(field.clone(), Value::Null);
969                            }
970                            break;
971                        }
972                    }
973                } else if !matches!(self.current_token, Token::Newline | Token::Eof)
974                    && matches!(self.current_token, Token::Delimiter(_))
975                {
976                    // Last field but there's another delimiter - too many values
977                    return Err(self
978                        .parse_error_with_context(format!(
979                            "Tabular row {}: expected {} values, but found extra values",
980                            row_index + 1,
981                            fields.len()
982                        ))
983                        .with_suggestion(format!(
984                            "Row {} should have exactly {} values",
985                            row_index + 1,
986                            fields.len()
987                        )));
988                }
989            }
990
991            if !self.options.strict && row.len() < fields.len() {
992                for field in fields.iter().skip(row.len()) {
993                    row.insert(field.clone(), Value::Null);
994                }
995            }
996
997            rows.push(Value::Object(row));
998
999            if matches!(self.current_token, Token::Eof) {
1000                break;
1001            }
1002
1003            if !matches!(self.current_token, Token::Newline) {
1004                if !self.options.strict {
1005                    while !matches!(self.current_token, Token::Newline | Token::Eof) {
1006                        self.advance()?;
1007                    }
1008                    if matches!(self.current_token, Token::Eof) {
1009                        break;
1010                    }
1011                } else {
1012                    return Err(self.parse_error_with_context(format!(
1013                        "Expected newline after tabular row {}",
1014                        row_index + 1
1015                    )));
1016                }
1017            }
1018
1019            if row_index + 1 < length {
1020                self.advance()?;
1021                if self.options.strict && matches!(self.current_token, Token::Newline) {
1022                    return Err(self.parse_error_with_context(
1023                        "Blank lines are not allowed inside tabular arrays in strict mode",
1024                    ));
1025                }
1026
1027                self.skip_newlines()?;
1028            } else if matches!(self.current_token, Token::Newline) {
1029                // After the last row, check if there are extra rows
1030                self.advance()?;
1031                self.skip_newlines()?;
1032
1033                let expected_indent = self.options.indent.get_spaces() * (depth + 1);
1034                let actual_indent = self.scanner.get_last_line_indent();
1035
1036                // If something at the same indent level, it might be a new row (error)
1037                // unless it's a key-value pair (which belongs to parent)
1038                if actual_indent == expected_indent && !matches!(self.current_token, Token::Eof) {
1039                    let is_key_value = matches!(self.current_token, Token::String(_, _))
1040                        && matches!(self.scanner.peek(), Some(':'));
1041
1042                    if !is_key_value {
1043                        return Err(self.parse_error_with_context(format!(
1044                            "Array length mismatch: expected {length} rows, but more rows found",
1045                        )));
1046                    }
1047                }
1048            }
1049        }
1050
1051        validation::validate_array_length(length, rows.len())?;
1052
1053        Ok(Value::Array(rows))
1054    }
1055
1056    fn parse_regular_array(&mut self, length: usize, depth: usize) -> ToonResult<Value> {
1057        let mut items = Vec::new();
1058
1059        // Empty arrays: return immediately without consuming the trailing newline,
1060        // so the caller's field-parsing loop can correctly check indentation.
1061        if length == 0 {
1062            return Ok(Value::Array(items));
1063        }
1064
1065        match &self.current_token {
1066            Token::Newline => {
1067                self.skip_newlines()?;
1068
1069                let expected_indent = self.options.indent.get_spaces() * (depth + 1);
1070
1071                for i in 0..length {
1072                    let current_indent = self.scanner.get_last_line_indent();
1073                    if self.options.strict {
1074                        self.validate_indentation(current_indent)?;
1075
1076                        if current_indent != expected_indent {
1077                            return Err(self.parse_error_with_context(format!(
1078                                "Invalid indentation for list item: expected {expected_indent} \
1079                                 spaces, found {current_indent}"
1080                            )));
1081                        }
1082                    }
1083                    if !matches!(self.current_token, Token::Dash) {
1084                        return Err(self
1085                            .parse_error_with_context(format!(
1086                                "Expected '-' for list item, found {:?}",
1087                                self.current_token
1088                            ))
1089                            .with_suggestion(format!(
1090                                "List arrays need '-' prefix for each item (item {} of {})",
1091                                i + 1,
1092                                length
1093                            )));
1094                    }
1095                    self.advance()?;
1096
1097                    let item_path = i.to_string();
1098                    self.layout_push(&item_path);
1099
1100                    let value = if matches!(self.current_token, Token::Newline | Token::Eof) {
1101                        Value::Object(Map::new())
1102                    } else if matches!(self.current_token, Token::LeftBracket) {
1103                        self.parse_array(depth + 1)?
1104                    } else if let Token::String(s, _) = &self.current_token {
1105                        let key = s.clone();
1106                        self.advance()?;
1107
1108                        if matches!(self.current_token, Token::Colon | Token::LeftBracket) {
1109                            // This is an object: key followed by colon or array bracket
1110                            // First field of list-item object may be an array requiring special
1111                            // indentation
1112                            self.layout_push(&key);
1113                            let first_value = if matches!(self.current_token, Token::LeftBracket) {
1114                                // Array directly after key (e.g., "- key[N]:")
1115                                // Use ListItemFirstField context to apply correct indentation
1116                                self.parse_array_with_context(
1117                                    depth + 1,
1118                                    ArrayParseContext::ListItemFirstField,
1119                                )?
1120                            } else {
1121                                self.advance()?;
1122                                // Handle nested arrays: "key: [2]: ..."
1123                                if matches!(self.current_token, Token::LeftBracket) {
1124                                    // Array after colon - not directly on hyphen line, use normal
1125                                    // context
1126                                    self.parse_array(depth + 2)?
1127                                } else {
1128                                    self.parse_field_value(depth + 2)?
1129                                }
1130                            };
1131                            self.layout_pop();
1132
1133                            let mut obj = Map::new();
1134                            obj.insert(key, first_value);
1135
1136                            let field_indent = self.options.indent.get_spaces() * (depth + 2);
1137
1138                            // Check if there are more fields at the same indentation level
1139                            let should_parse_more_fields =
1140                                if matches!(self.current_token, Token::Newline) {
1141                                    let next_indent = self.scanner.count_leading_spaces();
1142
1143                                    if next_indent < field_indent {
1144                                        false
1145                                    } else {
1146                                        self.advance()?;
1147
1148                                        if !self.options.strict {
1149                                            self.skip_newlines()?;
1150                                        }
1151                                        true
1152                                    }
1153                                } else if matches!(self.current_token, Token::String(_, _)) {
1154                                    // When already positioned at a field key, check its indent
1155                                    let current_indent = self.scanner.get_last_line_indent();
1156                                    current_indent == field_indent
1157                                } else {
1158                                    false
1159                                };
1160
1161                            // Parse additional fields if they're at the right indentation
1162                            if should_parse_more_fields {
1163                                while !matches!(self.current_token, Token::Eof) {
1164                                    let current_indent = self.scanner.get_last_line_indent();
1165
1166                                    if current_indent < field_indent {
1167                                        break;
1168                                    }
1169
1170                                    if current_indent != field_indent && self.options.strict {
1171                                        break;
1172                                    }
1173
1174                                    // Stop if we hit the next list item
1175                                    if matches!(self.current_token, Token::Dash) {
1176                                        break;
1177                                    }
1178
1179                                    let field_key = match &self.current_token {
1180                                        Token::String(s, _) => s.clone(),
1181                                        _ => break,
1182                                    };
1183                                    self.advance()?;
1184
1185                                    self.layout_push(&field_key);
1186                                    let field_value =
1187                                        if matches!(self.current_token, Token::LeftBracket) {
1188                                            self.parse_array(depth + 2)?
1189                                        } else if matches!(self.current_token, Token::Colon) {
1190                                            self.advance()?;
1191                                            if matches!(self.current_token, Token::LeftBracket) {
1192                                                self.parse_array(depth + 2)?
1193                                            } else {
1194                                                self.parse_field_value(depth + 2)?
1195                                            }
1196                                        } else {
1197                                            self.layout_pop();
1198                                            break;
1199                                        };
1200                                    self.layout_pop();
1201
1202                                    obj.insert(field_key, field_value);
1203
1204                                    if matches!(self.current_token, Token::Newline) {
1205                                        let next_indent = self.scanner.count_leading_spaces();
1206                                        if next_indent < field_indent {
1207                                            break;
1208                                        }
1209                                        self.advance()?;
1210                                        if !self.options.strict {
1211                                            self.skip_newlines()?;
1212                                        }
1213                                    } else if matches!(self.current_token, Token::String(_, _)) {
1214                                        // Tabular array parser already consumed the newline
1215                                        // and advanced to the next token — check indent
1216                                        let current_indent = self.scanner.get_last_line_indent();
1217                                        if current_indent != field_indent {
1218                                            break;
1219                                        }
1220                                    } else {
1221                                        break;
1222                                    }
1223                                }
1224                            }
1225
1226                            Value::Object(obj)
1227                        } else if matches!(self.current_token, Token::LeftBracket) {
1228                            // Array as object value: "key[2]: ..."
1229                            let array_value = self.parse_array(depth + 1)?;
1230                            let mut obj = Map::new();
1231                            obj.insert(key, array_value);
1232                            Value::Object(obj)
1233                        } else {
1234                            // Plain string value
1235                            Value::String(key)
1236                        }
1237                    } else {
1238                        self.parse_primitive()?
1239                    };
1240
1241                    self.layout_pop();
1242                    items.push(value);
1243
1244                    if items.len() < length {
1245                        if matches!(self.current_token, Token::Newline) {
1246                            self.advance()?;
1247
1248                            if self.options.strict && matches!(self.current_token, Token::Newline) {
1249                                return Err(self.parse_error_with_context(
1250                                    "Blank lines are not allowed inside list arrays in strict mode",
1251                                ));
1252                            }
1253
1254                            self.skip_newlines()?;
1255                        } else if !matches!(self.current_token, Token::Dash) {
1256                            return Err(self.parse_error_with_context(format!(
1257                                "Expected newline or next list item after list item {}",
1258                                i + 1
1259                            )));
1260                        }
1261                    } else if matches!(self.current_token, Token::Newline) {
1262                        // After the last item, check for extra items
1263                        self.advance()?;
1264                        self.skip_newlines()?;
1265
1266                        let list_indent = self.options.indent.get_spaces() * (depth + 1);
1267                        let actual_indent = self.scanner.get_last_line_indent();
1268                        // If we see another dash at the same indent, there are too many items
1269                        if actual_indent == list_indent && matches!(self.current_token, Token::Dash)
1270                        {
1271                            return Err(self.parse_error_with_context(format!(
1272                                "Array length mismatch: expected {length} items, but more items \
1273                                 found",
1274                            )));
1275                        }
1276                    }
1277                }
1278            }
1279            _ => {
1280                for i in 0..length {
1281                    if i > 0 {
1282                        if matches!(self.current_token, Token::Delimiter(_)) {
1283                            self.advance()?;
1284                        } else {
1285                            return Err(self
1286                                .parse_error_with_context(format!(
1287                                    "Expected delimiter, found {:?}",
1288                                    self.current_token
1289                                ))
1290                                .with_suggestion(format!(
1291                                    "Expected delimiter between items (item {} of {})",
1292                                    i + 1,
1293                                    length
1294                                )));
1295                        }
1296                    }
1297
1298                    let value = if matches!(self.current_token, Token::Delimiter(_))
1299                        || (matches!(self.current_token, Token::Eof | Token::Newline) && i < length)
1300                    {
1301                        Value::String(String::new())
1302                    } else if matches!(self.current_token, Token::LeftBracket) {
1303                        self.parse_array(depth + 1)?
1304                    } else {
1305                        self.parse_tabular_field_value()?
1306                    };
1307
1308                    items.push(value);
1309                }
1310            }
1311        }
1312
1313        validation::validate_array_length(length, items.len())?;
1314
1315        if self.options.strict && matches!(self.current_token, Token::Delimiter(_)) {
1316            return Err(self.parse_error_with_context(format!(
1317                "Array length mismatch: expected {length} items, but more items found",
1318            )));
1319        }
1320
1321        Ok(Value::Array(items))
1322    }
1323
1324    fn parse_tabular_field_value(&mut self) -> ToonResult<Value> {
1325        // Get the original text of the current token
1326        let token_text = self.scanner.last_token_text().to_string();
1327
1328        // Read remaining text until delimiter/newline/EOF
1329        let (rest, space_count) = self.scanner.read_until_delimiter_with_space_info();
1330
1331        if rest.is_empty() && space_count == 0 {
1332            // Single token — handle as primitive directly
1333            let result = match &self.current_token {
1334                Token::Null => Ok(Value::Null),
1335                Token::Bool(b) => Ok(Value::Bool(*b)),
1336                Token::SignedInteger(i) => Ok(Number::from(*i).into()),
1337                Token::UnsignedInteger(i) => Ok(Number::from(*i).into()),
1338                Token::Number(n) => {
1339                    let val = *n;
1340                    if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64 {
1341                        Ok(Number::from(val as i64).into())
1342                    } else {
1343                        Ok(Number::from_f64(val)
1344                            .ok_or_else(|| {
1345                                ToonError::InvalidInput(format!("Invalid number: {val}"))
1346                            })?
1347                            .into())
1348                    }
1349                }
1350                Token::String(s, _) => Ok(Value::String(s.clone())),
1351                _ => Err(self.parse_error_with_context(format!(
1352                    "Expected primitive value, found {:?}",
1353                    self.current_token
1354                ))),
1355            };
1356            self.advance()?;
1357            result
1358        } else {
1359            // Multiple tokens — combine original text + spaces + rest, then type-infer
1360            let mut value_str = token_text;
1361            for _ in 0..space_count {
1362                value_str.push(' ');
1363            }
1364            value_str.push_str(&rest);
1365
1366            let token = self.scanner.parse_value_string(&value_str)?;
1367            // Rescan so current_token is positioned at the next delimiter/newline
1368            self.current_token = self.scanner.scan_token()?;
1369            match token {
1370                Token::String(s, _) => Ok(Value::String(s)),
1371                Token::SignedInteger(i) => Ok(Number::from(i).into()),
1372                Token::UnsignedInteger(i) => Ok(Number::from(i).into()),
1373                Token::Number(n) => {
1374                    if n.is_finite() && n.fract() == 0.0 && n.abs() <= i64::MAX as f64 {
1375                        Ok(Number::from(n as i64).into())
1376                    } else {
1377                        Ok(Number::from_f64(n)
1378                            .ok_or_else(|| ToonError::InvalidInput(format!("Invalid number: {n}")))?
1379                            .into())
1380                    }
1381                }
1382                Token::Bool(b) => Ok(Value::Bool(b)),
1383                Token::Null => Ok(Value::Null),
1384                _ => Err(ToonError::InvalidInput("Unexpected token type".to_string())),
1385            }
1386        }
1387    }
1388
1389    fn parse_primitive(&mut self) -> ToonResult<Value> {
1390        match &self.current_token {
1391            Token::Null => {
1392                self.advance()?;
1393                Ok(Value::Null)
1394            }
1395            Token::Bool(b) => {
1396                let val = *b;
1397                self.advance()?;
1398                Ok(Value::Bool(val))
1399            }
1400            Token::SignedInteger(i) => {
1401                let val = *i;
1402                self.advance()?;
1403                Ok(Number::from(val).into())
1404            }
1405            Token::UnsignedInteger(i) => {
1406                let val = *i;
1407                self.advance()?;
1408                Ok(Number::from(val).into())
1409            }
1410            Token::Number(n) => {
1411                let val = *n;
1412                self.advance()?;
1413
1414                if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64 {
1415                    Ok(Number::from(val as i64).into())
1416                } else {
1417                    Ok(Number::from_f64(val)
1418                        .ok_or_else(|| ToonError::InvalidInput(format!("Invalid number: {val}")))?
1419                        .into())
1420                }
1421            }
1422            Token::String(s, _) => {
1423                let val = s.clone();
1424                self.advance()?;
1425                Ok(Value::String(val))
1426            }
1427            _ => Err(self.parse_error_with_context(format!(
1428                "Expected primitive value, found {:?}",
1429                self.current_token
1430            ))),
1431        }
1432    }
1433
1434    fn parse_error_with_context(&self, message: impl Into<String>) -> ToonError {
1435        let (line, column) = self.scanner.current_position();
1436        let message = message.into();
1437
1438        let context = self.get_error_context(line, column);
1439
1440        ToonError::ParseError {
1441            line,
1442            column,
1443            message,
1444            context: Some(Box::new(context)),
1445        }
1446    }
1447
1448    fn get_error_context(&self, line: usize, column: usize) -> ErrorContext {
1449        let lines: Vec<&str> = self.input.lines().collect();
1450
1451        let source_line = if line > 0 && line <= lines.len() {
1452            lines[line - 1].to_string()
1453        } else {
1454            String::new()
1455        };
1456
1457        let preceding_lines: Vec<String> = if line > 1 {
1458            lines[line.saturating_sub(3)..line - 1]
1459                .iter()
1460                .map(|s| s.to_string())
1461                .collect()
1462        } else {
1463            Vec::new()
1464        };
1465
1466        let following_lines: Vec<String> = if line < lines.len() {
1467            lines[line..line.saturating_add(2).min(lines.len())]
1468                .iter()
1469                .map(|s| s.to_string())
1470                .collect()
1471        } else {
1472            Vec::new()
1473        };
1474
1475        let indicator = if column > 0 {
1476            Some(format!("{:width$}^", "", width = column - 1))
1477        } else {
1478            None
1479        };
1480
1481        ErrorContext {
1482            source_line,
1483            preceding_lines,
1484            following_lines,
1485            suggestion: None,
1486            indicator,
1487        }
1488    }
1489
1490    fn validate_indentation(&self, indent_amount: usize) -> ToonResult<()> {
1491        if !self.options.strict {
1492            return Ok(());
1493        }
1494
1495        let indent_size = self.options.indent.get_spaces();
1496        // In strict mode, indentation must be a multiple of the configured indent size
1497        if indent_size > 0 && indent_amount > 0 && !indent_amount.is_multiple_of(indent_size) {
1498            Err(self.parse_error_with_context(format!(
1499                "Invalid indentation: found {indent_amount} spaces, but must be a multiple of \
1500                 {indent_size}"
1501            )))
1502        } else {
1503            Ok(())
1504        }
1505    }
1506}
1507
1508#[cfg(test)]
1509mod tests {
1510    use std::f64;
1511
1512    use serde_json::json;
1513
1514    use super::*;
1515
1516    fn parse(input: &str) -> ToonResult<Value> {
1517        let mut parser = Parser::new(input, DecodeOptions::default())?;
1518        parser.parse()
1519    }
1520
1521    #[test]
1522    fn test_parse_primitives() {
1523        assert_eq!(parse("null").unwrap(), json!(null));
1524        assert_eq!(parse("true").unwrap(), json!(true));
1525        assert_eq!(parse("false").unwrap(), json!(false));
1526        assert_eq!(parse("42").unwrap(), json!(42));
1527        assert_eq!(parse("3.141592653589793").unwrap(), json!(f64::consts::PI));
1528        assert_eq!(parse("hello").unwrap(), json!("hello"));
1529    }
1530
1531    #[test]
1532    fn test_parse_simple_object() {
1533        let result = parse("name: Alice\nage: 30").unwrap();
1534        assert_eq!(result["name"], json!("Alice"));
1535        assert_eq!(result["age"], json!(30));
1536    }
1537
1538    #[test]
1539    fn test_parse_primitive_array() {
1540        let result = parse("tags[3]: a,b,c").unwrap();
1541        assert_eq!(result["tags"], json!(["a", "b", "c"]));
1542    }
1543
1544    #[test]
1545    fn test_parse_empty_array() {
1546        let result = parse("items[0]:").unwrap();
1547        assert_eq!(result["items"], json!([]));
1548    }
1549
1550    #[test]
1551    fn test_parse_tabular_array() {
1552        let result = parse("users[2]{id,name}:\n  1,Alice\n  2,Bob").unwrap();
1553        assert_eq!(
1554            result["users"],
1555            json!([
1556                {"id": 1, "name": "Alice"},
1557                {"id": 2, "name": "Bob"}
1558            ])
1559        );
1560    }
1561
1562    #[test]
1563    fn test_empty_tokens() {
1564        let result = parse("items[3]: a,,c").unwrap();
1565        assert_eq!(result["items"], json!(["a", "", "c"]));
1566    }
1567
1568    #[test]
1569    fn test_empty_nested_object() {
1570        let result = parse("user:").unwrap();
1571        assert_eq!(result, json!({"user": {}}));
1572    }
1573
1574    #[test]
1575    fn test_list_item_object() {
1576        let result =
1577            parse("items[2]:\n  - id: 1\n    name: First\n  - id: 2\n    name: Second").unwrap();
1578        assert_eq!(
1579            result["items"],
1580            json!([
1581                {"id": 1, "name": "First"},
1582                {"id": 2, "name": "Second"}
1583            ])
1584        );
1585    }
1586
1587    #[test]
1588    fn test_nested_array_in_list_item() {
1589        let result = parse("items[1]:\n  - tags[3]: a,b,c").unwrap();
1590        assert_eq!(result["items"], json!([{"tags": ["a", "b", "c"]}]));
1591    }
1592
1593    #[test]
1594    fn test_two_level_siblings() {
1595        let input = "x:\n  y: 1\n  z: 2";
1596        let opts = DecodeOptions::default();
1597        let mut parser = Parser::new(input, opts).unwrap();
1598        let result = parser.parse().unwrap();
1599
1600        let x = result.as_object().unwrap().get("x").unwrap();
1601        let x_obj = x.as_object().unwrap();
1602
1603        assert_eq!(x_obj.len(), 2, "x should have 2 keys");
1604        assert_eq!(x_obj.get("y").unwrap(), &serde_json::json!(1));
1605        assert_eq!(x_obj.get("z").unwrap(), &serde_json::json!(2));
1606    }
1607
1608    #[test]
1609    fn test_nested_object_with_sibling() {
1610        let input = "a:\n  b:\n    c: 1\n  d: 2";
1611        let opts = DecodeOptions::default();
1612        let mut parser = Parser::new(input, opts).unwrap();
1613        let result = parser.parse().unwrap();
1614
1615        let a = result.as_object().unwrap().get("a").unwrap();
1616        let a_obj = a.as_object().unwrap();
1617
1618        assert_eq!(a_obj.len(), 2, "a should have 2 keys (b and d)");
1619        assert!(a_obj.contains_key("b"), "a should have key 'b'");
1620        assert!(a_obj.contains_key("d"), "a should have key 'd'");
1621
1622        let b = a_obj.get("b").unwrap().as_object().unwrap();
1623        assert_eq!(b.len(), 1, "b should have only 1 key (c)");
1624        assert!(b.contains_key("c"), "b should have key 'c'");
1625        assert!(!b.contains_key("d"), "b should NOT have key 'd'");
1626    }
1627
1628    #[test]
1629    fn test_field_value_with_parentheses() {
1630        let result = parse("msg: Mostly Functions (3 of 3)").unwrap();
1631        assert_eq!(result, json!({"msg": "Mostly Functions (3 of 3)"}));
1632
1633        let result = parse("val: (hello)").unwrap();
1634        assert_eq!(result, json!({"val": "(hello)"}));
1635
1636        let result = parse("test: a (b) c (d)").unwrap();
1637        assert_eq!(result, json!({"test": "a (b) c (d)"}));
1638    }
1639
1640    #[test]
1641    fn test_field_value_number_with_parentheses() {
1642        let result = parse("code: 0(f)").unwrap();
1643        assert_eq!(result, json!({"code": "0(f)"}));
1644
1645        let result = parse("val: 5(test)").unwrap();
1646        assert_eq!(result, json!({"val": "5(test)"}));
1647
1648        let result = parse("msg: test 123)").unwrap();
1649        assert_eq!(result, json!({"msg": "test 123)"}));
1650    }
1651
1652    #[test]
1653    fn test_field_value_single_token_optimization() {
1654        let result = parse("name: hello").unwrap();
1655        assert_eq!(result, json!({"name": "hello"}));
1656
1657        let result = parse("age: 42").unwrap();
1658        assert_eq!(result, json!({"age": 42}));
1659
1660        let result = parse("active: true").unwrap();
1661        assert_eq!(result, json!({"active": true}));
1662
1663        let result = parse("value: null").unwrap();
1664        assert_eq!(result, json!({"value": null}));
1665    }
1666
1667    #[test]
1668    fn test_field_value_multi_token() {
1669        let result = parse("msg: hello world").unwrap();
1670        assert_eq!(result, json!({"msg": "hello world"}));
1671
1672        let result = parse("msg: test 123 end").unwrap();
1673        assert_eq!(result, json!({"msg": "test 123 end"}));
1674    }
1675
1676    #[test]
1677    fn test_field_value_spacing_preserved() {
1678        let result = parse("val: hello world").unwrap();
1679        assert_eq!(result, json!({"val": "hello world"}));
1680
1681        let result = parse("val: 0(f)").unwrap();
1682        assert_eq!(result, json!({"val": "0(f)"}));
1683    }
1684
1685    #[test]
1686    fn test_round_trip_parentheses() {
1687        use crate::{
1688            decode::decode_default,
1689            encode::encode_default,
1690        };
1691
1692        let original = json!({
1693            "message": "Mostly Functions (3 of 3)",
1694            "code": "0(f)",
1695            "simple": "(hello)",
1696            "mixed": "test 123)"
1697        });
1698
1699        let encoded = encode_default(&original).unwrap();
1700        let decoded: Value = decode_default(&encoded).unwrap();
1701
1702        assert_eq!(original, decoded);
1703    }
1704
1705    #[test]
1706    fn test_multiple_fields_with_edge_cases() {
1707        let input = r#"message: Mostly Functions (3 of 3)
1708sone: (hello)
1709hello: 0(f)"#;
1710
1711        let result = parse(input).unwrap();
1712        assert_eq!(
1713            result,
1714            json!({
1715                "message": "Mostly Functions (3 of 3)",
1716                "sone": "(hello)",
1717                "hello": "0(f)"
1718            })
1719        );
1720    }
1721
1722    #[test]
1723    fn test_decode_list_item_tabular_array_v3() {
1724        // Tabular arrays as first field of list items
1725        // Rows must be at depth +2 relative to hyphen (6 spaces from root)
1726        let input = r#"items[1]:
1727  - users[2]{id,name}:
1728      1,Ada
1729      2,Bob
1730    status: active"#;
1731
1732        let result = parse(input).unwrap();
1733
1734        assert_eq!(
1735            result,
1736            json!({
1737                "items": [
1738                    {
1739                        "users": [
1740                            {"id": 1, "name": "Ada"},
1741                            {"id": 2, "name": "Bob"}
1742                        ],
1743                        "status": "active"
1744                    }
1745                ]
1746            })
1747        );
1748    }
1749
1750    #[test]
1751    fn test_decode_list_item_tabular_array_multiple_items() {
1752        // Multiple list items each with tabular array as first field
1753        let input = r#"data[2]:
1754  - records[1]{id,val}:
1755      1,x
1756    count: 1
1757  - records[1]{id,val}:
1758      2,y
1759    count: 1"#;
1760
1761        let result = parse(input).unwrap();
1762
1763        assert_eq!(
1764            result,
1765            json!({
1766                "data": [
1767                    {
1768                        "records": [{"id": 1, "val": "x"}],
1769                        "count": 1
1770                    },
1771                    {
1772                        "records": [{"id": 2, "val": "y"}],
1773                        "count": 1
1774                    }
1775                ]
1776            })
1777        );
1778    }
1779
1780    #[test]
1781    fn test_decode_list_item_tabular_array_with_multiple_fields() {
1782        // List item with tabular array first and multiple sibling fields
1783        let input = r#"entries[1]:
1784  - people[2]{name,age}:
1785      Alice,30
1786      Bob,25
1787    total: 2
1788    category: staff"#;
1789
1790        let result = parse(input).unwrap();
1791
1792        assert_eq!(
1793            result,
1794            json!({
1795                "entries": [
1796                    {
1797                        "people": [
1798                            {"name": "Alice", "age": 30},
1799                            {"name": "Bob", "age": 25}
1800                        ],
1801                        "total": 2,
1802                        "category": "staff"
1803                    }
1804                ]
1805            })
1806        );
1807    }
1808
1809    #[test]
1810    fn test_decode_list_item_non_tabular_array_unchanged() {
1811        // Non-tabular arrays as first field should work normally
1812        let input = r#"items[1]:
1813  - tags[3]: a,b,c
1814    name: test"#;
1815
1816        let result = parse(input).unwrap();
1817
1818        assert_eq!(
1819            result,
1820            json!({
1821                "items": [
1822                    {
1823                        "tags": ["a", "b", "c"],
1824                        "name": "test"
1825                    }
1826                ]
1827            })
1828        );
1829    }
1830
1831    #[test]
1832    fn test_decode_strict_rejects_v2_tabular_indent() {
1833        use crate::decode::decode_strict;
1834
1835        // Old format: rows at depth +1 (4 spaces from root)
1836        // Strict mode should reject this incorrect indentation
1837        let input_v2 = r#"items[1]:
1838  - users[2]{id,name}:
1839    1,Ada
1840    2,Bob"#;
1841
1842        let result = decode_strict::<Value>(input_v2);
1843
1844        // Should error due to incorrect indentation
1845        assert!(
1846            result.is_err(),
1847            "Old format with incorrect indentation should be rejected in strict mode"
1848        );
1849        let err_msg = result.unwrap_err().to_string();
1850        assert!(
1851            err_msg.contains("indentation") || err_msg.contains("Invalid indentation"),
1852            "Error should mention indentation. Got: {}",
1853            err_msg
1854        );
1855    }
1856
1857    #[test]
1858    fn test_decode_tabular_array_not_in_list_item_unchanged() {
1859        // Regular tabular arrays (not in list items) should still use depth +1
1860        let input = r#"users[2]{id,name}:
1861  1,Ada
1862  2,Bob"#;
1863
1864        let result = parse(input).unwrap();
1865
1866        assert_eq!(
1867            result,
1868            json!({
1869                "users": [
1870                    {"id": 1, "name": "Ada"},
1871                    {"id": 2, "name": "Bob"}
1872                ]
1873            })
1874        );
1875    }
1876
1877    #[test]
1878    fn test_decode_nested_tabular_not_first_field() {
1879        // Tabular array as a subsequent field (not first) should use normal depth
1880        let input = r#"items[1]:
1881  - name: test
1882    data[2]{id,val}:
1883      1,x
1884      2,y"#;
1885
1886        let result = parse(input).unwrap();
1887
1888        assert_eq!(
1889            result,
1890            json!({
1891                "items": [
1892                    {
1893                        "name": "test",
1894                        "data": [
1895                            {"id": 1, "val": "x"},
1896                            {"id": 2, "val": "y"}
1897                        ]
1898                    }
1899                ]
1900            })
1901        );
1902    }
1903
1904    #[test]
1905    fn test_array_element_number_followed_by_string() {
1906        // Issue #56: Array elements starting with a number should be parsed as string
1907        // when followed by non-numeric text
1908        let result = parse("version1[1]: 1.0 something").unwrap();
1909        assert_eq!(result["version1"], json!(["1.0 something"]));
1910
1911        let result = parse("data[1]: 42 units").unwrap();
1912        assert_eq!(result["data"], json!(["42 units"]));
1913
1914        // Pure numbers should still be parsed as numbers
1915        let result = parse("nums[1]: 42").unwrap();
1916        assert_eq!(result["nums"], json!([42]));
1917
1918        let result = parse("nums[1]: 2.75").unwrap();
1919        assert_eq!(result["nums"], json!([2.75]));
1920    }
1921
1922    #[test]
1923    fn test_issue_59_multiple_spaces_preserved() {
1924        // Issue #59: Multiple spaces between words should be preserved
1925        // Field value context
1926        let result = parse("key: a   b").unwrap();
1927        assert_eq!(result["key"], json!("a   b"));
1928
1929        // Tabular cell context
1930        let result = parse("data[2]: a   b, c   d").unwrap();
1931        assert_eq!(result["data"], json!(["a   b", "c   d"]));
1932
1933        // Root-level value
1934        let result = parse("a   b").unwrap();
1935        assert_eq!(result, json!("a   b"));
1936    }
1937
1938    #[test]
1939    fn test_issue_60_mixed_type_tokens_as_string() {
1940        // Issue #60: "1 null" and "a 1" should parse as strings in tabular rows
1941        // Tabular cell context
1942        let result = parse("data[2]: 1 null, a 1").unwrap();
1943        assert_eq!(result["data"], json!(["1 null", "a 1"]));
1944
1945        // Root-level value
1946        let result = parse("1 null").unwrap();
1947        assert_eq!(result, json!("1 null"));
1948
1949        let result = parse("a 1").unwrap();
1950        assert_eq!(result, json!("a 1"));
1951
1952        // Field value context
1953        let result = parse("key: 1 null").unwrap();
1954        assert_eq!(result["key"], json!("1 null"));
1955
1956        let result = parse("key: a 1").unwrap();
1957        assert_eq!(result["key"], json!("a 1"));
1958    }
1959
1960    #[test]
1961    fn test_issue_61_number_format_preserved() {
1962        // Issue #61: "1.0 b" should preserve "1.0", not become "1 b"
1963        // Tabular cell context
1964        let result = parse("data[2]: 1.0 b, 1e1 b").unwrap();
1965        assert_eq!(result["data"], json!(["1.0 b", "1e1 b"]));
1966
1967        // Field value context
1968        let result = parse("key: 1.0 b").unwrap();
1969        assert_eq!(result["key"], json!("1.0 b"));
1970
1971        let result = parse("key: 1e1 b").unwrap();
1972        assert_eq!(result["key"], json!("1e1 b"));
1973
1974        // Root-level value
1975        let result = parse("1.0 b").unwrap();
1976        assert_eq!(result, json!("1.0 b"));
1977
1978        let result = parse("1e1 b").unwrap();
1979        assert_eq!(result, json!("1e1 b"));
1980    }
1981}