tabnas_csv/lib.rs
1// Copyright (c) 2021-2026 Richard Rodger, MIT License
2
3// The engine's error carries a code, position, hint and a formatted
4// report, so it is large by design and `Result<_, TabnasError>` trips
5// clippy's `result_large_err`. The engine allows the lint at its own
6// crate root for the same reason, and so does tabnas-jsonic; boxing here
7// would make `parse` return a different shape from `Tabnas::parse` and
8// from the other two ports.
9#![allow(clippy::result_large_err)]
10
11//! The CSV grammar plugin for the `tabnas` parsing engine.
12//!
13//! CSV text becomes an array of objects (one per row, keyed by the header
14//! row) or an array of arrays, with RFC 4180 `""`-escaped quoting, custom
15//! field and record separators, streaming, and a strict / non-strict mode.
16//! In strict mode (the default) every field is raw text; in non-strict
17//! mode a field body can hold embedded jsonic (`[1,2]`, `{x:1}`).
18//!
19//! The plugin is not standalone: it layers on the relaxed-JSON grammar of
20//! [`tabnas_jsonic`] (the `val` / `map` / `list` / `pair` / `elem` rules)
21//! and reuses the engine's lexer, comment handling and lifecycle hooks,
22//! exactly as the canonical TypeScript plugin in `ts/src/csv.ts` does.
23//!
24//! ```
25//! let value = tabnas_csv::parse("name,age\nAlice,30\nBob,25")?;
26//! assert_eq!(
27//! value.to_json().to_string(),
28//! r#"[{"name":"Alice","age":"30"},{"name":"Bob","age":"25"}]"#
29//! );
30//! # Ok::<(), tabnas_csv::CsvError>(())
31//! ```
32//!
33//! TypeScript is canonical: `ts/src/csv.ts` defines behaviour and option
34//! defaults, and `csv-grammar.jsonic` at the repository root is the
35//! grammar every runtime embeds. The shared fixtures in `test/spec/*.tsv`
36//! are the parity contract across TypeScript, Go and Rust.
37
38use std::cell::RefCell;
39use std::fmt;
40use std::rc::Rc;
41use std::sync::{Arc, OnceLock};
42
43use indexmap::IndexMap;
44use serde::{Deserialize, Serialize};
45use serde_json::json;
46use tabnas::{
47 ActionError, AltSpec, Context, GrammarSetting, GrammarSpec, ImperativeLexMatcher, Options,
48 Plugin, PluginError, Rule, RuleSpec, Tabnas, Tin, Token, Value, TIN_BD, TIN_ST,
49};
50
51/// This crate's version. It MUST equal `ts/package.json` "version": the
52/// release orchestrator rewrites both, and `tests/version_test.rs` fails
53/// the build if they drift. Mirrors `VERSION` in `ts/src/csv.ts` and
54/// `const VERSION` in `go/csv.go`.
55pub const VERSION: &str = "0.6.9";
56
57/// The README's Rust examples run as doctests, so a stale one fails the
58/// gate rather than misleading the reader. Its `toml` and `bash` fences
59/// are skipped; rustdoc runs only the `rust` ones.
60#[cfg(doctest)]
61#[doc = include_str!("../README.md")]
62mod readme_examples {}
63
64/// The error a failed parse produces, re-exported so callers need not
65/// depend on the engine crate directly. Its `code` is the contract the
66/// shared fixtures pin: `csv_extra_field`, `csv_missing_field`, and the
67/// engine's own `unexpected` and `unterminated_string`.
68pub use tabnas::TabnasError as CsvError;
69
70// --- BEGIN EMBEDDED csv-grammar.jsonic ---
71const GRAMMAR_TEXT: &str = r##"
72# CSV Grammar Definition
73# Parsed by a standard Jsonic instance and passed to jsonic.grammar()
74# Function references (@ prefixed) are resolved against the refs map
75#
76# Token naming:
77# #LN - line ending (removed from per-instance IGNORE set)
78# #SP - whitespace (removed from per-instance IGNORE set in strict mode)
79# #CA - comma / field separator
80# #ZZ - end of input
81# #VAL - token set: text, string, number, value literals
82#
83# Rules csv, newline, record, text are fully defined here.
84# Rules list, elem, val are modified in code (strict mode defines from scratch;
85# non-strict prepends to existing defaults to preserve JSON parsing).
86
87{
88 options: rule: { start: csv }
89 options: lex: { emptyResult: [] }
90 # Placeholders are {braces}: the engine's error injector (strinject in
91 # @tabnas/parser) substitutes {name} from the details object passed to
92 # token.bad(). A $name is left in the message verbatim.
93 options: error: {
94 csv_extra_field: 'unexpected extra field value: {fsrc}'
95 csv_missing_field: 'missing field'
96 }
97 options: hint: {
98 csv_extra_field: 'Row {row} has too many fields (the first of which is: {fsrc}). Only {len}\nfields per row are expected.'
99 csv_missing_field: 'Row {row} has too few fields. {len} fields per row are expected.'
100 }
101
102 rule: csv: open: [
103 { s: '#ZZ' }
104 { s: '#LN' p: newline c: '@not-record-empty' }
105 { p: record }
106 ]
107
108 rule: newline: open: [
109 { s: '#LN #LN' r: newline }
110 { s: '#LN' r: newline }
111 { s: '#ZZ' }
112 { r: record }
113 ]
114 rule: newline: close: [
115 { s: '#LN #LN' r: newline g: end }
116 { s: '#LN' r: newline g: end }
117 { s: '#ZZ' g: end }
118 { r: record }
119 ]
120
121 rule: record: open: [
122 { p: list }
123 ]
124 rule: record: close: [
125 { s: '#ZZ' g: end }
126 { s: '#LN #ZZ' b: 1 g: end }
127 { s: '#LN' r: '@record-close-next' g: end }
128 ]
129
130 rule: text: open: [
131 { s: ['#VAL' '#SP'] b: 1 r: text n: { text: 1 } g: 'csv,space,follows' a: '@text-follows' }
132 { s: ['#SP' '#VAL'] r: text n: { text: 1 } g: 'csv,space,leads' a: '@text-leads' }
133 { s: ['#SP' '#CA #LN #ZZ'] b: 1 n: { text: 1 } g: 'csv,end' a: '@text-end' }
134 { s: '#SP' n: { text: 1 } g: 'csv,space' a: '@text-space' p: '@text-space-push' }
135 {}
136 ]
137}
138"##;
139// --- END EMBEDDED csv-grammar.jsonic ---
140
141/// The grammar source every runtime embeds, verbatim: the text of
142/// `csv-grammar.jsonic` at the repository root. Exposed so a test can hold
143/// the embedded copy to the file on disk.
144pub fn grammar_text() -> &'static str {
145 GRAMMAR_TEXT
146}
147
148/// The name every csv plugin registers under, and so the namespace of its
149/// plugin options (`parser.plugin_options("csv")`).
150const PLUGIN_NAME: &str = "csv";
151
152// ---------------------------------------------------------------------------
153// Named function references.
154//
155// The grammar names every closure, and the closures are registered on the
156// instance BEFORE the document is installed, because the document is what
157// looks them up. The lifecycle names follow the engine's `@<rule>-<phase>`
158// convention and are wired onto their rule automatically.
159// ---------------------------------------------------------------------------
160
161const CSV_BO: &str = "@csv-bo";
162const CSV_AC: &str = "@csv-ac";
163const RECORD_BC: &str = "@record-bc";
164const TEXT_BC: &str = "@text-bc";
165const TEXT_FOLLOWS: &str = "@text-follows";
166const TEXT_LEADS: &str = "@text-leads";
167const TEXT_END: &str = "@text-end";
168const TEXT_SPACE: &str = "@text-space";
169const NOT_RECORD_EMPTY: &str = "@not-record-empty";
170const RECORD_CLOSE_NEXT: &str = "@record-close-next";
171const TEXT_SPACE_PUSH: &str = "@text-space-push";
172/// The `options.lex.match.stringcsv.make` reference: the RFC 4180 string
173/// matcher factory.
174const CSV_STRING: &str = "@csv-string";
175
176/// The matcher's priority. Below the engine's own bands (fixed literals sit
177/// at 2e6, quoted strings at 5e6), so a `""`-escaped field is cut before
178/// the standard string matcher can see its opening quote. The same order
179/// the TypeScript and Go plugins register.
180const CSV_STRING_ORDER: f64 = 100_000.0;
181
182// ---------------------------------------------------------------------------
183// Options
184// ---------------------------------------------------------------------------
185
186/// What a streaming parse reports, in order: `Start`, one `Record` per
187/// data row, then `End`. The counterpart of the TypeScript `stream(what,
188/// record)` callback's `'start'`, `'record'` and `'end'` events.
189#[derive(Debug, Clone, PartialEq)]
190pub enum StreamEvent {
191 /// The parse has begun.
192 Start,
193 /// One parsed record, shaped by `object` exactly as it would have been
194 /// stored in the result.
195 Record(Value),
196 /// The parse has finished.
197 End,
198}
199
200/// A streaming callback: records are handed to it as they are parsed and
201/// are not stored in the result, which comes back empty.
202#[derive(Clone)]
203pub struct Stream(Arc<dyn Fn(StreamEvent) + Send + Sync>);
204
205impl Stream {
206 /// Wrap a callback.
207 pub fn new(callback: impl Fn(StreamEvent) + Send + Sync + 'static) -> Self {
208 Stream(Arc::new(callback))
209 }
210
211 fn send(&self, event: StreamEvent) {
212 (self.0)(event);
213 }
214}
215
216impl fmt::Debug for Stream {
217 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
218 formatter.write_str("Stream(<function>)")
219 }
220}
221
222/// Field options. See [`CsvOptions`].
223#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
224#[serde(default)]
225pub struct FieldOptions {
226 /// The field separator, when it is not a comma. May be several
227 /// characters (`"~~"`).
228 pub separation: Option<String>,
229 /// The key prefix for a field beyond the header's columns: the third
230 /// field of a two-column document is `field~2`.
231 pub nonameprefix: String,
232 /// The value an empty field takes. Any JSON value; the default is the
233 /// empty string.
234 pub empty: serde_json::Value,
235 /// Explicit column names, used in place of a header row.
236 pub names: Option<Vec<String>>,
237 /// Fail the parse with `csv_extra_field` or `csv_missing_field` when a
238 /// row's field count differs from the header's (or from
239 /// `names.len()`). Inert without a known field list.
240 pub exact: bool,
241}
242
243impl Default for FieldOptions {
244 fn default() -> Self {
245 FieldOptions {
246 separation: None,
247 nonameprefix: "field~".to_string(),
248 empty: serde_json::Value::String(String::new()),
249 names: None,
250 exact: false,
251 }
252 }
253}
254
255/// Record options. See [`CsvOptions`].
256#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
257#[serde(default)]
258pub struct RecordOptions {
259 /// The record separator characters, when they are not the default
260 /// `\n` / `\r\n` / `\r`.
261 pub separators: Option<String>,
262 /// Keep an empty line as a record of empty fields, rather than
263 /// skipping it.
264 pub empty: bool,
265}
266
267/// String options. See [`CsvOptions`].
268#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
269#[serde(default)]
270pub struct StringOptions {
271 /// The quote character of a quoted field.
272 pub quote: String,
273 /// Whether to use the RFC 4180 `""`-escaping string matcher. `None`
274 /// means on in strict mode and off in non-strict mode; `Some(false)`
275 /// in strict mode leaves quoted fields to the jsonic string matcher
276 /// with its backslash escapes.
277 pub csv: Option<bool>,
278}
279
280impl Default for StringOptions {
281 fn default() -> Self {
282 StringOptions {
283 quote: "\"".to_string(),
284 csv: None,
285 }
286 }
287}
288
289/// The plugin options: the counterpart of the TypeScript `CsvOptions` and
290/// `Csv.defaults`, and of the Go `Defaults` map. [`Default`] is the
291/// documented default set.
292///
293/// The three-state options (`trim`, `comment`, `number`, `value`) are
294/// `Option<bool>`: `None` is the TypeScript `null`, which means "off in
295/// strict mode, on in non-strict mode".
296#[derive(Debug, Clone, Serialize, Deserialize)]
297#[serde(default)]
298pub struct CsvOptions {
299 /// Drop the whitespace around an unquoted field.
300 pub trim: Option<bool>,
301 /// Lex `#` comments, so a comment line is no record and a trailing
302 /// comment is not field text.
303 pub comment: Option<bool>,
304 /// Lex numbers, so `1e2` is the number `100` rather than text.
305 pub number: Option<bool>,
306 /// Lex the value keywords, so `true`, `false` and `null` are values
307 /// rather than text.
308 pub value: Option<bool>,
309 /// The first row names the columns.
310 pub header: bool,
311 /// Each record is an object keyed by column name; `false` gives an
312 /// array of field values.
313 pub object: bool,
314 /// Every field is raw text. `false` lets a field body hold embedded
315 /// jsonic (`[1,2]`, `{x:1}`, a backslash-escaped string) and turns
316 /// `trim`, `comment`, `number` and `value` on unless set.
317 pub strict: bool,
318 /// Field options.
319 pub field: FieldOptions,
320 /// Record options.
321 pub record: RecordOptions,
322 /// String options.
323 pub string: StringOptions,
324 /// Stream records to a callback instead of collecting them. A closure
325 /// cannot travel in the plugin option bag, so it is carried here,
326 /// outside serialization, and reaches the plugin through
327 /// [`plugin_with`] and [`make_with`].
328 #[serde(skip)]
329 pub stream: Option<Stream>,
330}
331
332impl Default for CsvOptions {
333 fn default() -> Self {
334 CsvOptions {
335 trim: None,
336 comment: None,
337 number: None,
338 value: None,
339 header: true,
340 object: true,
341 strict: true,
342 field: FieldOptions::default(),
343 record: RecordOptions::default(),
344 string: StringOptions::default(),
345 stream: None,
346 }
347 }
348}
349
350impl CsvOptions {
351 /// Read options out of a plugin option bag, the form
352 /// [`Tabnas::use_plugin`] merges and hands to the plugin. A missing key
353 /// takes its default; an unknown key is ignored, as in the other two
354 /// runtimes.
355 pub fn from_value(value: &Value) -> Result<Self, String> {
356 serde_json::from_value(value.to_json()).map_err(|error| format!("csv options: {error}"))
357 }
358
359 /// The options as a plugin option bag, the form [`Plugin::with_defaults`]
360 /// and [`Tabnas::use_plugin`] take. The `stream` callback does not
361 /// travel; see [`plugin_with`].
362 pub fn to_value(&self) -> Value {
363 let json = serde_json::to_value(self).expect("the option struct serializes");
364 Value::from_json(&json)
365 }
366}
367
368/// The settings the closures capture: the options normalised the way the
369/// TypeScript plugin normalises them at the top of `Csv`.
370#[derive(Clone)]
371struct Settings {
372 strict: bool,
373 objres: bool,
374 header: bool,
375 trim: bool,
376 record_empty: bool,
377 empty: Value,
378 nonameprefix: String,
379 names: Option<Vec<String>>,
380 exact: bool,
381 stream: Option<Stream>,
382}
383
384// ---------------------------------------------------------------------------
385// Node helpers
386// ---------------------------------------------------------------------------
387
388/// Assign a rule's node: the Rust spelling of TypeScript `r.node = v`.
389///
390/// A pushed or replaced rule SHARES its parent's node cell, so writing
391/// through `rule.node.borrow_mut()` would overwrite the parent's node too.
392/// Assigning installs a fresh cell instead. The cell is borrowed directly
393/// only to mutate a container the rule genuinely shares: pushing a field
394/// onto the enclosing record, or a record onto the document.
395fn set_node(rule: &mut Rule, value: Value) {
396 rule.node = Rc::new(RefCell::new(value));
397}
398
399fn list_push(node: &mut Value, value: Value) {
400 match node {
401 Value::Array(items) => Arc::make_mut(items).push(value),
402 Value::ListRef(list) => Arc::make_mut(list).value.push(value),
403 _ => {}
404 }
405}
406
407fn items_of(node: &Value) -> Vec<Value> {
408 match node {
409 Value::Array(items) => items.as_ref().clone(),
410 Value::ListRef(list) => list.value.clone(),
411 _ => Vec::new(),
412 }
413}
414
415fn flag(rule: &Rule, name: &str) -> bool {
416 matches!(rule.u.get(name), Some(Value::Bool(true)))
417}
418
419fn counter(rule: &Rule, name: &str) -> i32 {
420 rule.n.get(name).copied().unwrap_or(0)
421}
422
423/// JavaScript's `Number::toString` (ECMA-262 6.1.6.1.20), which is what
424/// the canonical TypeScript `'' + value` spells, and so what a numeric cell
425/// becomes as field text or as a column name.
426///
427/// Rust's own `f64` formatting differs from it in three ways that reach a
428/// header key. It keeps the sign of negative zero (`-0`, where JavaScript
429/// says `0`). It never switches to exponent form, where JavaScript does so
430/// at `1e21` and at `1e-7`. And `format!("{n:.0}")` prints a large integral
431/// float's exact binary value (`123456789012345683968`) rather than its
432/// shortest round-tripping digits (`123456789012345680000`).
433fn js_number_to_string(number: f64) -> String {
434 if number.is_nan() {
435 return "NaN".to_string();
436 }
437 // Catches -0.0 as well: JavaScript spells both zeros "0".
438 if number == 0.0 {
439 return "0".to_string();
440 }
441 if number < 0.0 {
442 return format!("-{}", js_number_to_string(-number));
443 }
444 if number.is_infinite() {
445 return "Infinity".to_string();
446 }
447
448 // The specification wants the shortest digit string `s` that round-trips
449 // (length `k`), and `n`, the position of the decimal point relative to
450 // it. Rust's `{:e}` yields digits of exactly that shortest length.
451 let shortest = format!("{number:e}");
452 let shortest_k = shortest
453 .split_once('e')
454 .map(|(mantissa, _)| mantissa.chars().filter(char::is_ascii_digit).count())
455 .expect("a finite f64 always formats with an exponent");
456
457 // Re-render to that same length to settle a tie. Where two digit
458 // strings of length `k` are equally close to `number`, the
459 // specification takes the one ending in an even digit; Rust's shortest
460 // form does not, but its exactly-rounded fixed-precision form does.
461 let exponential = format!("{:.*e}", shortest_k - 1, number);
462 let (mantissa, exponent) = exponential
463 .split_once('e')
464 .expect("a finite f64 always formats with an exponent");
465 // Rounding can leave trailing zeros (and, on a carry, one digit too
466 // many); dropping them keeps `s` shortest, which is what `k` means.
467 let digits = mantissa
468 .chars()
469 .filter(|digit| *digit != '.')
470 .collect::<String>();
471 let digits = digits.trim_end_matches('0');
472 let digits = if digits.is_empty() { "0" } else { digits };
473 let k = digits.len() as i32;
474 let n = exponent
475 .parse::<i32>()
476 .expect("a formatted exponent is an integer")
477 + 1;
478
479 // The four cases of the specification, in its order. The range bounds
480 // are `k <= n <= 21`, `0 < n <= 21` and `-6 < n <= 0`.
481 if (k..=21).contains(&n) {
482 // Integral, with n - k trailing zeros to restore.
483 let mut text = digits.to_string();
484 text.push_str(&"0".repeat((n - k) as usize));
485 text
486 } else if (1..=21).contains(&n) {
487 let point = n as usize;
488 format!("{}.{}", &digits[..point], &digits[point..])
489 } else if (-5..=0).contains(&n) {
490 format!("0.{}{}", "0".repeat(-n as usize), digits)
491 } else {
492 // Exponent form. `n - 1` is never 0 here, so the sign is never "+0".
493 let sign = if n - 1 < 0 { '-' } else { '+' };
494 let power = (n - 1).abs();
495 if k == 1 {
496 format!("{digits}e{sign}{power}")
497 } else {
498 format!("{}.{}e{sign}{power}", &digits[..1], &digits[1..])
499 }
500 }
501}
502
503/// A JavaScript `'' + value`: what the text actions concatenate. Text and
504/// string tokens carry their text; a number or keyword (`number` /
505/// `value` on) spells itself the way JavaScript does.
506fn value_text(value: &Value) -> String {
507 match value {
508 Value::Undefined => String::new(),
509 Value::Null => "null".to_string(),
510 Value::Bool(flag) => flag.to_string(),
511 Value::Number(number) => js_number_to_string(*number),
512 Value::String(text) => text.clone(),
513 Value::Text(text) => text.string.clone(),
514 other => other.to_json().to_string(),
515 }
516}
517
518/// A token's value as text (`r.o0.val`), or its source when it carries no
519/// value at all.
520fn token_text(token: &Token) -> String {
521 if token.val.is_undefined() {
522 token.src.as_str().to_string()
523 } else {
524 value_text(&token.val)
525 }
526}
527
528/// A header cell as a column name: `obj[cell] = ...` applies
529/// ToPropertyKey, which for anything but a symbol is ToString.
530///
531/// Called from the one place the canonical converts a cell, the object
532/// shaping in [`record_before_close`], and never on the header row
533/// itself, which is stored raw. A reading that takes no name from a cell
534/// -- `object: false`, `header: false`, a header-only document, or a
535/// `field.exact` mismatch, which is checked first -- never reaches this.
536///
537/// `None` means JavaScript cannot make a primitive of the value at all,
538/// which is every OBJECT. A jsonic object is allocated with a null
539/// prototype, so it inherits neither `toString` nor `Symbol.toPrimitive`
540/// and the canonical runtime throws
541/// `TypeError: Cannot convert object to primitive value` instead of
542/// naming the column. This port cannot raise a JavaScript `TypeError`, so
543/// it refuses the document; `DIVERGENCE.md` records that choice.
544///
545/// That holds for a PARSED object. An object supplied as an OPTION value
546/// is the other case, and the canonical answers it differently: the
547/// option merge rebuilds a plain source object onto `Object.prototype` on
548/// the way into the bag, so `String` names the column `[object Object]`
549/// there and nothing throws, even for one the caller made with
550/// `Object.create(null)`.
551///
552/// `from_option` is which of the two this cell is, and the caller knows
553/// because there is exactly one option that can put a value in a header
554/// row: `field.empty`, dropped into a syntactically empty cell before any
555/// rule runs, so `,a` under `{field: {empty: <value>}}` takes its first
556/// column name from it. The flag is set where the options are READ rather
557/// than guessed from the value, because a `Value::Object` is the same
558/// type either way, and it travels INTO [`join_text`], so an object
559/// nested in an option array is answered the same as one at the top.
560///
561/// Falling through to [`value_text`] is what made the refusal necessary
562/// in the first place: its last-resort arm renders the value as JSON,
563/// which named the column `{"x":1.0}` for `{x:1}` and `[1.0,2.0]` for
564/// `[1,2]`, neither of which the canonical runtime produces for any
565/// input.
566fn key_text(value: &Value, from_option: bool) -> Option<String> {
567 match value {
568 Value::Undefined
569 | Value::Null
570 | Value::Bool(_)
571 | Value::Number(_)
572 | Value::String(_)
573 | Value::Text(_) => Some(value_text(value)),
574 Value::Array(items) => join_text(items, from_option),
575 Value::Object(_) if from_option => Some("[object Object]".to_string()),
576 // A PARSED object, and the two reference wrappers the engine keeps
577 // for its own bookkeeping, have no ToString to apply.
578 _ => None,
579 }
580}
581
582/// Whether two values are the SAME container, rather than two containers
583/// that compare equal. It is how a cell is recognised as the one
584/// `field.empty` supplied: an option value is cloned into the cell, and a
585/// clone of a container is a refcount bump on the same allocation, while
586/// the parser allocates its own. Equality would not do, because a parsed
587/// `{q:1}` equals an option `{q:1}` and only one of them may be named.
588fn same_container(value: &Value, other: &Value) -> bool {
589 match (value, other) {
590 (Value::Object(left), Value::Object(right)) => Arc::ptr_eq(left, right),
591 (Value::Array(left), Value::Array(right)) => Arc::ptr_eq(left, right),
592 _ => false,
593 }
594}
595
596/// `Array.prototype.toString`, which is `join(',')` with no separator
597/// argument (ECMA-262 23.1.3.17 and 23.1.3.34): every element is
598/// converted by the same rules, `null` and `undefined` become the empty
599/// string, and a nested array joins recursively, so `[1,[2,3]]` flattens
600/// to `1,2,3` and `[]` is the empty string.
601///
602/// A null ELEMENT is the empty string while a null CELL is `null`: the
603/// empty string comes from `join`, not from ToString, so it applies only
604/// inside an array.
605///
606/// The recursion needs no depth bound and no seen-set. A parsed value is
607/// a TREE: the engine folds each finished rule's value into its parent
608/// and never stores a reference to an ancestor, so no element can reach
609/// its own array and the walk always terminates. Its depth is the
610/// document's bracket nesting, which `tabnas-jsonic` already bounds at
611/// 127 containers with the parse budget this plugin inherits.
612///
613/// An OPTION value does reach this, through `field.empty`, and it cannot
614/// contain itself either, which is where the Go port differs.
615/// `field.empty` is a `serde_json::Value` and `field.names` a
616/// `Vec<String>`, both owned acyclic trees. The engine's own `Value`
617/// shares containers behind an `Arc`, but building a cycle out of one
618/// needs a second handle on a container while its refcount is still 1,
619/// which the type does not allow. Go's option is an unconstrained `any`,
620/// a caller can spell `a := make([]any, 1); a[0] = a`, and there the same
621/// walk had to be bounded.
622///
623/// `from_option` travels from [`key_text`], so an object nested in an
624/// option array is named `[object Object]` where a parsed one refuses.
625fn join_text(items: &[Value], from_option: bool) -> Option<String> {
626 let mut joined = String::new();
627 for (index, item) in items.iter().enumerate() {
628 if 0 < index {
629 joined.push(',');
630 }
631 match item {
632 Value::Null | Value::Undefined => {}
633 other => joined.push_str(&key_text(other, from_option)?),
634 }
635 }
636 Some(joined)
637}
638
639fn number(value: usize) -> Value {
640 Value::Number(value as f64)
641}
642
643// ---------------------------------------------------------------------------
644// The closures the grammar names
645// ---------------------------------------------------------------------------
646
647/// The text of the previous text rule's node: the accumulator a chained
648/// text rule (`r: text`) extends.
649fn prev_text(rule: &Rule) -> String {
650 rule.prev_rule
651 .as_ref()
652 .map(|prev| value_text(&prev.node.borrow()))
653 .unwrap_or_default()
654}
655
656/// `r.node = v.node = text`, where `v` is the rule itself on the first
657/// text rule of a chain and the previous one afterwards.
658fn assign_text(rule: &mut Rule, chained: bool, text: String) {
659 let value = Value::String(text);
660 if chained {
661 if let Some(prev) = rule.prev_rule.as_ref() {
662 *prev.node.borrow_mut() = value.clone();
663 }
664 }
665 set_node(rule, value);
666}
667
668/// Text then space: `x ` continues a field, so keep the text and read on.
669fn text_follows(rule: &mut Rule, _context: &mut Context) -> Result<(), ActionError> {
670 let chained = counter(rule, "text") != 1;
671 let head = if chained {
672 prev_text(rule)
673 } else {
674 String::new()
675 };
676 let text = head + &rule.o0().map(token_text).unwrap_or_default();
677 assign_text(rule, chained, text);
678 Ok(())
679}
680
681/// Space then text: the space is field text unless it leads a trimmed
682/// field.
683fn text_leads(trim: bool) -> impl Fn(&mut Rule, &mut Context) -> Result<(), ActionError> {
684 move |rule, _context| {
685 let count = counter(rule, "text");
686 let chained = count != 1;
687 let head = if chained {
688 prev_text(rule)
689 } else {
690 String::new()
691 };
692 let space = if 2 <= count || !trim {
693 rule.o0().map(|token| token.src.as_str().to_string())
694 } else {
695 None
696 };
697 let text = head
698 + &space.unwrap_or_default()
699 + &rule
700 .o1()
701 .map(|token| token.src.as_str().to_string())
702 .unwrap_or_default();
703 assign_text(rule, chained, text);
704 Ok(())
705 }
706}
707
708/// Trailing space before a separator, line end or end of source: field
709/// text unless trimmed.
710fn text_end(trim: bool) -> impl Fn(&mut Rule, &mut Context) -> Result<(), ActionError> {
711 move |rule, _context| {
712 let chained = counter(rule, "text") != 1;
713 let head = if chained {
714 prev_text(rule)
715 } else {
716 String::new()
717 };
718 let space = if trim {
719 None
720 } else {
721 rule.o0().map(|token| token.src.as_str().to_string())
722 };
723 assign_text(rule, chained, head + &space.unwrap_or_default());
724 Ok(())
725 }
726}
727
728/// A lone space: in strict mode it is field text (unless trimmed); in
729/// non-strict mode the action does nothing and the alternate pushes `val`
730/// to read an embedded value.
731fn text_space(
732 strict: bool,
733 trim: bool,
734) -> impl Fn(&mut Rule, &mut Context) -> Result<(), ActionError> {
735 let end = text_end(trim);
736 move |rule, context| {
737 if strict {
738 end(rule, context)
739 } else {
740 Ok(())
741 }
742 }
743}
744
745/// `text` before-close: hand the field text (or the embedded value a
746/// pushed `val` produced) to the enclosing `val`. This is `r.parent.node =
747/// ...` in the canonical grammar, and it writes into the parent's cell.
748fn text_before_close(rule: &mut Rule, _context: &mut Context) -> Result<(), ActionError> {
749 let value = if rule.child_node.is_undefined() {
750 rule.node.borrow().clone()
751 } else {
752 rule.child_node.clone()
753 };
754 if let Some(parent) = rule.parent_node.as_ref() {
755 *parent.borrow_mut() = value;
756 }
757 Ok(())
758}
759
760fn record_index(context: &Context) -> usize {
761 match context.u.get("recordI") {
762 Some(Value::Number(index)) => *index as usize,
763 _ => 0,
764 }
765}
766
767/// `record` before-close: the header row names the columns; every later
768/// row becomes a record, checked against the field count under
769/// `field.exact`, shaped by `object`, and either streamed or pushed onto
770/// the document.
771fn record_before_close(
772 settings: Settings,
773) -> impl Fn(
774 &mut Rule,
775 &mut Context,
776 Option<&tabnas::RuleSnapshot>,
777 Option<Token>,
778) -> Result<Option<Token>, ActionError> {
779 move |rule, context, _next, out| {
780 let record_i = record_index(context);
781 // The field list is held as the RAW cells, the way the canonical
782 // holds `ctx.u.fields`, because a cell becomes a column NAME only
783 // where an object record is built. `field.names` widens into the
784 // same shape so the two sources stay interchangeable.
785 // `fields_from_options` is the provenance of the WHOLE list: a
786 // header row is parsed, `field.names` is an option. A parsed
787 // header row can still hold an option value, because `field.empty`
788 // is dropped into an empty cell before any rule runs, so the flag
789 // is taken per cell below as well.
790 let (fields, fields_from_options): (Option<Vec<Value>>, bool) =
791 match context.u.get("fields") {
792 Some(Value::Array(names)) => (Some(names.as_ref().clone()), false),
793 _ => (
794 settings.names.as_ref().map(|names| {
795 names
796 .iter()
797 .map(|name| Value::String(name.clone()))
798 .collect()
799 }),
800 true,
801 ),
802 };
803
804 if record_i == 0 && settings.header {
805 // Kept exactly as parsed, which is what the canonical keeps:
806 // `ctx.u.fields = r.child.node`. Nothing is converted here,
807 // and an empty list is still a field list, as it is in
808 // JavaScript where `[]` is truthy.
809 context.u.insert(
810 "fields".to_string(),
811 Value::array(items_of(&rule.child_node)),
812 );
813 } else {
814 let mut record = items_of(&rule.child_node);
815
816 // The field-count check is independent of the result SHAPE: it
817 // only needs a known field list (from the header row or
818 // `field.names`), so it fires for arrays as well as objects.
819 if let Some(fields) = fields.as_ref() {
820 if settings.exact && record.len() != fields.len() {
821 let code = if record.len() > fields.len() {
822 "csv_extra_field"
823 } else {
824 "csv_missing_field"
825 };
826 // The messages interpolate {row}, {len} and {fsrc}, so
827 // the details are supplied. `row` counts from 1 and
828 // includes the header line, so it matches the line
829 // number a spreadsheet or editor reports.
830 let mut details = vec![
831 ("row".to_string(), number(record_i + 1)),
832 ("len".to_string(), number(fields.len())),
833 ];
834 if let Some(extra) = record.get(fields.len()) {
835 details.push(("fsrc".to_string(), extra.clone()));
836 }
837 let mut token = context.t0().cloned().unwrap_or_else(|| {
838 Token::new("#BD", TIN_BD, Value::Undefined, "", Default::default())
839 });
840 token.bad_with_details(code, details);
841 return Ok(Some(token));
842 }
843 }
844
845 let filled = |cell: Option<&Value>| match cell {
846 Some(value) if !value.is_undefined() => value.clone(),
847 _ => settings.empty.clone(),
848 };
849
850 let shaped = if settings.objres {
851 let mut object = IndexMap::new();
852 let mut index = 0;
853 if let Some(fields) = fields.as_ref() {
854 for (position, cell) in fields.iter().enumerate() {
855 // This is the ONE place a header cell becomes a
856 // column name, and the only place the canonical
857 // converts one: `obj[fields[fI]] = ...` puts the
858 // raw cell through ToPropertyKey, which for
859 // anything but a symbol is ToString.
860 //
861 // A PARSED object has no ToString at all, and
862 // the canonical runtime throws a TypeError here
863 // rather than naming the column. This port cannot
864 // raise one, so it refuses the document with the
865 // engine's inherited `unexpected` code instead of
866 // inventing a name; `DIVERGENCE.md` records that
867 // choice. Refusing at the header row instead was
868 // too early: it also refused an `object: false`
869 // parse and a header-only document, neither of
870 // which ever asks for a name, and both of which
871 // the canonical returns a value for.
872 //
873 // An object an OPTION supplied is named
874 // `[object Object]`, which is what the canonical
875 // does for it, and `from_option` is how the two
876 // are told apart. The type cannot tell them
877 // apart: both are `Value::Object`.
878 let from_option =
879 fields_from_options || same_container(cell, &settings.empty);
880 let Some(name) = key_text(cell, from_option) else {
881 let mut token = context.t0().cloned().unwrap_or_else(|| {
882 Token::new("#BD", TIN_BD, Value::Undefined, "", Default::default())
883 });
884 token.bad("unexpected");
885 return Ok(Some(token));
886 };
887 object.insert(name, filled(record.get(position)));
888 }
889 index = fields.len();
890 }
891 while index < record.len() {
892 object.insert(
893 format!("{}{index}", settings.nonameprefix),
894 filled(record.get(index)),
895 );
896 index += 1;
897 }
898 Value::object(object)
899 } else {
900 for cell in record.iter_mut() {
901 if cell.is_undefined() {
902 *cell = settings.empty.clone();
903 }
904 }
905 Value::array(record)
906 };
907
908 match settings.stream.as_ref() {
909 Some(stream) => stream.send(StreamEvent::Record(shaped)),
910 None => list_push(&mut rule.node.borrow_mut(), shaped),
911 }
912 }
913
914 context
915 .u
916 .insert("recordI".to_string(), number(record_i + 1));
917 Ok(out)
918 }
919}
920
921/// Register every closure the grammar names, and the string matcher
922/// factory. Before the documents, because the documents look them up.
923fn register_refs(parser: &mut Tabnas, settings: &Settings, quote: &str) {
924 let stream = settings.stream.clone();
925 parser.state_action_ref(CSV_BO, move |rule, context| {
926 context.u.insert("recordI".to_string(), number(0));
927 if let Some(stream) = stream.as_ref() {
928 stream.send(StreamEvent::Start);
929 }
930 set_node(rule, Value::array(Vec::new()));
931 Ok(())
932 });
933
934 let stream = settings.stream.clone();
935 parser.state_action_ref(CSV_AC, move |_rule, _context| {
936 if let Some(stream) = stream.as_ref() {
937 stream.send(StreamEvent::End);
938 }
939 Ok(())
940 });
941
942 parser.state_action_with_next_ref(RECORD_BC, record_before_close(settings.clone()));
943 parser.state_action_ref(TEXT_BC, text_before_close);
944
945 parser.action_with_context(TEXT_FOLLOWS, text_follows);
946 parser.action_with_context(TEXT_LEADS, text_leads(settings.trim));
947 parser.action_with_context(TEXT_END, text_end(settings.trim));
948 parser.action_with_context(TEXT_SPACE, text_space(settings.strict, settings.trim));
949
950 let record_empty = settings.record_empty;
951 parser.alt_condition(NOT_RECORD_EMPTY, move |_rule, _context| !record_empty);
952 parser.alt_replace(RECORD_CLOSE_NEXT, move |_rule, _context| {
953 Some(if record_empty { "record" } else { "newline" }.to_string())
954 });
955
956 let strict = settings.strict;
957 parser.alt_push(TEXT_SPACE_PUSH, move |_rule, _context| {
958 Some(if strict { "" } else { "val" }.to_string())
959 });
960
961 parser.lex_match_factory_ref(CSV_STRING, csv_string_matcher(quote));
962}
963
964// ---------------------------------------------------------------------------
965// The RFC 4180 string matcher
966// ---------------------------------------------------------------------------
967
968/// The custom CSV string matcher factory: `"a""b"` is the text `a"b`.
969///
970/// A reduced copy of the standard jsonic string matcher, with the doubled
971/// quote as the only escape. It is the counterpart of the TypeScript
972/// `buildCsvStringMatcher` export and the Go `BuildCsvStringMatcher`, in
973/// the shape the engine's `options.lex.match.<name>.make` reference takes:
974/// register it with [`Tabnas::lex_match_factory_ref`] and name it from an
975/// options document.
976///
977/// ```
978/// use tabnas::GrammarSpec;
979///
980/// let mut parser = tabnas_jsonic::make();
981/// parser.lex_match_factory_ref("@csv-string", tabnas_csv::csv_string_matcher("\""));
982/// let spec = GrammarSpec::from_value(serde_json::json!({
983/// "options": { "lex": { "match": {
984/// "stringcsv": { "order": 100000, "make": "@csv-string" },
985/// } } },
986/// }))?;
987/// parser.grammar(&spec)?;
988/// assert_eq!(parser.parse(r#""a""b""#)?, tabnas::Value::String("a\"b".to_string()));
989/// # Ok::<(), Box<dyn std::error::Error>>(())
990/// ```
991///
992/// The matcher runs only where the engine wants a string token, and only
993/// at the quote character. A line character inside the quotes is field
994/// text (a quoted field may span lines). Any other control character is
995/// field text too when the engine's `string.allowControl` option is on,
996/// as the CSV plugin sets it, and `unprintable` when it is off, as it is
997/// by default on a plain instance. A quote left open at the end of the
998/// source is `unterminated_string`. Loop exhaustion is what detects the
999/// open quote, so an odd number of quotes (`"""`, `"""""`) is caught
1000/// rather than read as a terminated string.
1001///
1002/// A quote that is not exactly one UTF-16 code unit leaves the matcher
1003/// inert, and the factory answers `None` rather than installing one. The
1004/// canonical opens a quoted field with
1005/// `quoteMap = { [options.string.quote]: true }` and tests
1006/// `quoteMap[src[sI]]`, where `src[sI]` is ONE UTF-16 code unit, so it
1007/// can only ever fire for a quote that is one code unit: the empty
1008/// string, a multi-character quote such as `ab`, and an astral character
1009/// such as U+1F600, which JavaScript holds as a surrogate PAIR, are all
1010/// inert there. Go carries the same guard, with the same reasoning.
1011pub fn csv_string_matcher(
1012 quote: impl Into<String>,
1013) -> impl Fn(&Options) -> Option<ImperativeLexMatcher> + Send + Sync + 'static {
1014 let quote: String = quote.into();
1015 let single_code_unit = {
1016 let mut characters = quote.chars();
1017 match (characters.next(), characters.next()) {
1018 (Some(character), None) => character.len_utf16() == 1,
1019 _ => false,
1020 }
1021 };
1022 move |options: &Options| {
1023 if !single_code_unit {
1024 return None;
1025 }
1026 let quote = quote.clone();
1027 let line_chars: Vec<char> = options.line.chars.chars().collect();
1028 let allow_control = options.string.allow_control;
1029 Some(Arc::new(
1030 move |lexer: &mut tabnas::Lexer<'_>, _rule: &mut Rule, _context: &mut Context| {
1031 if !lexer.window().starts_with(quote.as_str()) {
1032 return None;
1033 }
1034 let point = lexer.point();
1035 // The field is scanned in the text the lexer holds: the
1036 // whole source, or in a parse from a reader the window
1037 // (`lex.window`, admin ADR-30). A field the window ends
1038 // inside is not unterminated until the source is: the
1039 // window is asked for twice as much and the field scanned
1040 // again, so a long field costs its length, not its square.
1041 let (text, at) = loop {
1042 let rest = lexer.window();
1043 match scan_quoted(rest, quote.as_str(), &line_chars, allow_control) {
1044 Quoted::Closed { text, at } => break (text, at),
1045 Quoted::QuoteAtEnd { text, at } => {
1046 let held = rest[..at].chars().count();
1047 lexer.fill(held + 1);
1048 if lexer.window().len() > at {
1049 continue;
1050 }
1051 break (text, at);
1052 }
1053 Quoted::Control { at } => {
1054 // Any other control character is field text
1055 // when the engine's `string.allowControl` is
1056 // on, which the plugin turns on; only with it
1057 // off is the character refused. The diagnostic
1058 // points at the control character, as it does
1059 // in TypeScript.
1060 let count = rest[..at].chars().count();
1061 lexer.advance_chars(count);
1062 return Some(lexer.bad("unprintable"));
1063 }
1064 Quoted::Open { at } => {
1065 let held = rest[..at].chars().count();
1066 // `fill` says whether that much lies ahead; the
1067 // source may end sooner and still have given
1068 // more, which is scanned again.
1069 lexer.fill(held.saturating_mul(2).max(held + 1));
1070 if lexer.window().len() > at {
1071 continue;
1072 }
1073 let start = point.site.pos;
1074 return Some(lexer.bad_span(
1075 "unterminated_string",
1076 start,
1077 start + held,
1078 ));
1079 }
1080 }
1081 };
1082
1083 let source = lexer.window()[..at].to_string();
1084 let token = lexer.token("#ST", TIN_ST, Value::String(text), source.as_str(), point);
1085 lexer.advance_chars(source.chars().count());
1086 Some(token)
1087 },
1088 ))
1089 }
1090}
1091
1092/// Where a quoted field's scan ended, in the text it was given.
1093enum Quoted {
1094 /// The closing quote: the field's text, and the byte just past it.
1095 Closed { text: String, at: usize },
1096 /// A quote that is the last character the text holds: the field's text
1097 /// before it, and the byte just past it.
1098 QuoteAtEnd { text: String, at: usize },
1099 /// A control character the field may not hold, at this byte.
1100 Control { at: usize },
1101 /// The text ended first, at this byte: the field is unterminated, or
1102 /// goes on past what the lexer holds.
1103 Open { at: usize },
1104}
1105
1106/// Scan the quoted field `rest` starts with, its opening quote included: a
1107/// doubled quote is one literal quote, a single one closes the field.
1108fn scan_quoted(rest: &str, quote: &str, line_chars: &[char], allow_control: bool) -> Quoted {
1109 let mut text = String::new();
1110 let mut at = quote.len();
1111 while at < rest.len() {
1112 let tail = &rest[at..];
1113 if tail.starts_with(quote) {
1114 at += quote.len();
1115 if rest[at..].starts_with(quote) {
1116 text.push_str(quote);
1117 at += quote.len();
1118 continue;
1119 }
1120 if at == rest.len() {
1121 // The quote is the last character held: it closes the field
1122 // if the source ends here, and is half of a doubled quote if
1123 // another follows past what the lexer holds.
1124 return Quoted::QuoteAtEnd { text, at };
1125 }
1126 return Quoted::Closed { text, at };
1127 }
1128 let character = tail.chars().next().expect("inside the text");
1129 if !line_chars.contains(&character) && (character as u32) < 32 && !allow_control {
1130 return Quoted::Control { at };
1131 }
1132 text.push(character);
1133 at += character.len_utf8();
1134 }
1135 Quoted::Open { at }
1136}
1137
1138// ---------------------------------------------------------------------------
1139// The documents
1140// ---------------------------------------------------------------------------
1141
1142/// Whole numbers as integers. The grammar text is parsed by jsonic, whose
1143/// numbers are all `f64`, and the engine's document loader reads an
1144/// alternate's `b` and `n` as integers.
1145fn integral(value: &mut serde_json::Value) {
1146 match value {
1147 serde_json::Value::Number(number) => {
1148 if let Some(float) = number.as_f64() {
1149 if float.fract() == 0.0 && float.abs() < 9.0e15 {
1150 if float < 0.0 {
1151 *value = serde_json::Value::from(float as i64);
1152 } else {
1153 *value = serde_json::Value::from(float as u64);
1154 }
1155 }
1156 }
1157 }
1158 serde_json::Value::Array(items) => items.iter_mut().for_each(integral),
1159 serde_json::Value::Object(entries) => entries.values_mut().for_each(integral),
1160 _ => {}
1161 }
1162}
1163
1164/// The embedded grammar as a serialized document: parsed by a standard
1165/// jsonic instance, as the TypeScript `new Tabnas().use(jsonic).parse(
1166/// grammarText)` and the Go `parseGrammarText` do.
1167fn grammar_document() -> Result<serde_json::Value, PluginError> {
1168 let parsed = tabnas_jsonic::parse(GRAMMAR_TEXT).map_err(|error| {
1169 PluginError(format!("csv: the embedded grammar does not parse: {error}"))
1170 })?;
1171 let mut document = parsed.to_json();
1172 if !document.is_object() || document.get("rule").is_none() {
1173 return Err(PluginError(
1174 "csv: the embedded grammar has no `rule` table".to_string(),
1175 ));
1176 }
1177 integral(&mut document);
1178 Ok(document)
1179}
1180
1181/// The per-instance engine options, mirroring `jsonicOptions` in
1182/// `ts/src/csv.ts`. The static options (`rule.start`, `lex.emptyResult`,
1183/// `error`, `hint`) live in the grammar text.
1184fn options_document(
1185 options: &CsvOptions,
1186 settings: &Settings,
1187 flags: &LexFlags,
1188) -> serde_json::Value {
1189 let strict = settings.strict;
1190
1191 // Fixed-token overrides: in strict mode the JSON structural tokens and
1192 // the `:` key separator are switched off; a configured field separator
1193 // rebinds `#CA`.
1194 let mut fixed = serde_json::Map::new();
1195 if strict {
1196 for name in ["#OB", "#CB", "#OS", "#CS", "#CL"] {
1197 fixed.insert(name.to_string(), serde_json::Value::Null);
1198 }
1199 }
1200 if let Some(separation) = options
1201 .field
1202 .separation
1203 .as_deref()
1204 .filter(|separation| !separation.is_empty())
1205 {
1206 fixed.insert("#CA".to_string(), json!(separation));
1207 }
1208
1209 // IGNORE set: `#LN` leaves it so row breaks are significant; in strict
1210 // mode `#SP` leaves it too so whitespace inside a field survives. This
1211 // engine merges a token set INDEX-WISE onto the one already installed,
1212 // as the canonical TypeScript deep merge does: a `null` drops that
1213 // position and a shorter array keeps the tail of the set it overlays.
1214 // So the override is spelled position for position over the jsonic
1215 // default `[#SP, #LN, #CM]`, exactly as `ts/src/csv.ts` spells it, and
1216 // the trailing position is named rather than left off: an override that
1217 // stops short of the installed length would keep whatever the tail
1218 // holds, and this one has to answer the same on a derived instance,
1219 // where the set it overlays is already the reduced one.
1220 let ignore = if strict {
1221 json!([null, null, "#CM"])
1222 } else {
1223 json!(["#SP", null, "#CM"])
1224 };
1225
1226 let mut line = json!({ "single": settings.record_empty });
1227 if let Some(separators) = options.record.separators.as_deref() {
1228 line["chars"] = json!(separators);
1229 line["rowChars"] = json!(separators);
1230 }
1231
1232 let mut document = json!({
1233 "options": {
1234 "rule": { "exclude": if strict { "jsonic,imp" } else { "imp" } },
1235 "fixed": { "token": fixed },
1236 "tokenSet": { "IGNORE": ignore },
1237 "number": { "lex": flags.number },
1238 "value": { "lex": flags.value },
1239 "comment": { "lex": flags.comment },
1240 // A quoted field holds every character but an unescaped quote,
1241 // the C0 controls included, as other CSV readers accept them:
1242 // the CSV renderer writes a cell holding a TAB, an ASCII 30 or
1243 // an ASCII 31 between quotes, because CSV has no other spelling
1244 // for it. The jsonic string matcher, which reads the quote
1245 // characters the CSV matcher does not own and every quote under
1246 // `string.csv: false` or in non-strict mode, refuses a raw
1247 // control as `unprintable` unless this is on, and
1248 // `csv_string_matcher` honours the same option.
1249 "string": { "allowControl": true },
1250 "line": line,
1251 },
1252 });
1253 if flags.csv_strings {
1254 document["options"]["lex"] = json!({
1255 "match": {
1256 "stringcsv": { "order": CSV_STRING_ORDER, "make": CSV_STRING },
1257 },
1258 });
1259 }
1260 document
1261}
1262
1263/// The lexer switches the mode decides.
1264struct LexFlags {
1265 number: bool,
1266 value: bool,
1267 comment: bool,
1268 csv_strings: bool,
1269}
1270
1271fn apply_document(parser: &mut Tabnas, document: serde_json::Value) -> Result<(), PluginError> {
1272 let spec = GrammarSpec::from_value(document).map_err(|error| PluginError(error.0))?;
1273 parser
1274 .grammar(&spec)
1275 .map_err(|error| PluginError(error.0))?;
1276 Ok(())
1277}
1278
1279// ---------------------------------------------------------------------------
1280// The list / elem / val rules, modified in code
1281// ---------------------------------------------------------------------------
1282
1283/// An alternate over token sequences, with the fields the CSV rules use.
1284fn alt(s: Vec<Vec<Tin>>, b: usize, g: &str) -> AltSpec {
1285 let mut alt = AltSpec::new();
1286 alt.s = s;
1287 alt.b = b;
1288 alt.g = g.to_string();
1289 alt
1290}
1291
1292fn push_alt(name: &str) -> AltSpec {
1293 let mut alt = AltSpec::new();
1294 alt.p = Some(name.to_string());
1295 alt
1296}
1297
1298fn with_push(mut alt: AltSpec, name: &str) -> AltSpec {
1299 alt.p = Some(name.to_string());
1300 alt
1301}
1302
1303fn with_replace(mut alt: AltSpec, name: &str) -> AltSpec {
1304 alt.r = Some(name.to_string());
1305 alt
1306}
1307
1308/// `rs.open([...])` / `rs.close([...])` with no `append`: the whole list
1309/// goes in front of the existing alternates, in its own order.
1310fn prepend_open(spec: &mut RuleSpec, alts: Vec<AltSpec>) {
1311 for alt in alts.into_iter().rev() {
1312 spec.prepend_open(alt);
1313 }
1314}
1315
1316fn prepend_close(spec: &mut RuleSpec, alts: Vec<AltSpec>) {
1317 for alt in alts.into_iter().rev() {
1318 spec.prepend_close(alt);
1319 }
1320}
1321
1322/// The tokens the CSV rules match.
1323struct Tokens {
1324 ln: Tin,
1325 ca: Tin,
1326 sp: Tin,
1327 zz: Tin,
1328 val: Vec<Tin>,
1329}
1330
1331/// An empty element before a separator: `,` pushes the empty value and
1332/// marks the element done, so `elem` before-close does not push again.
1333fn empty_before_comma(empty: Value) -> impl Fn(&mut Rule, &mut Context) + Send + Sync + 'static {
1334 move |rule, _context| {
1335 list_push(&mut rule.node.borrow_mut(), empty.clone());
1336 rule.u_mut().insert("done".to_string(), Value::Bool(true));
1337 }
1338}
1339
1340/// An empty element at the end of the line: `,\n` / `,<eof>`.
1341fn empty_at_end(empty: Value) -> impl Fn(&mut Rule, &mut Context) + Send + Sync + 'static {
1342 move |rule, _context| {
1343 list_push(&mut rule.node.borrow_mut(), empty.clone());
1344 }
1345}
1346
1347/// Allocate the per-record field array. Under the engine's node model a
1348/// pushed child rule is seeded with the parent's node, and jsonic's
1349/// `list` before-open only allocates for an implicit list, which the CSV
1350/// `record -> list` path never is: without this the fields would land on
1351/// the inherited document array.
1352fn allocate_fields(rule: &mut Rule, _context: &mut Context) {
1353 set_node(rule, Value::array(Vec::new()));
1354}
1355
1356/// Strict mode: `list`, `elem` and `val` are replaced entirely. The JSON
1357/// structural tokens are off, so only the CSV alternates are needed.
1358fn strict_rules(parser: &mut Tabnas, tokens: &Tokens, empty: &Value) {
1359 let Tokens {
1360 ln,
1361 ca,
1362 sp,
1363 zz,
1364 val,
1365 } = tokens;
1366 let (ln, ca, sp, zz) = (*ln, *ca, *sp, *zz);
1367
1368 parser.define_rule("list", |spec| {
1369 spec.clear();
1370 spec.add_bo(allocate_fields);
1371 // Do not consume the LN that closes an empty record.
1372 spec.add_open(alt(vec![vec![ln]], 1, ""));
1373 spec.add_open(push_alt("elem"));
1374 // LN ends the record.
1375 spec.add_close(alt(vec![vec![ln]], 1, "end"));
1376 spec.add_close(alt(vec![vec![zz]], 0, "end"));
1377 });
1378
1379 let before_comma = empty.clone();
1380 let at_end = empty.clone();
1381 parser.define_rule("elem", move |spec| {
1382 spec.clear();
1383 let mut comma = alt(vec![vec![ca]], 1, "empty");
1384 comma.add_action(empty_before_comma(before_comma));
1385 spec.add_open(comma);
1386 spec.add_open(push_alt("val"));
1387
1388 let mut trailing = alt(vec![vec![ca], vec![ln, zz]], 1, "comma,empty");
1389 trailing.add_action(empty_at_end(at_end));
1390 spec.add_close(trailing);
1391 // Next element.
1392 spec.add_close(with_replace(alt(vec![vec![ca]], 0, "comma"), "elem"));
1393 // LN ends the record.
1394 spec.add_close(alt(vec![vec![ln]], 1, "end"));
1395 spec.add_close(alt(vec![vec![zz]], 0, "end"));
1396
1397 // Push the parsed field value, unless an open alternate already did.
1398 spec.add_bc(|rule, _context| {
1399 if !flag(rule, "done") && !rule.child_node.is_undefined() {
1400 let child = rule.child_node.clone();
1401 list_push(&mut rule.node.borrow_mut(), child);
1402 }
1403 });
1404 });
1405
1406 let val = val.clone();
1407 parser.define_rule("val", move |spec| {
1408 spec.clear();
1409 // Reset the parent-seeded node so an empty value resolves to no
1410 // value, not to the inherited field array.
1411 spec.add_bo(|rule, _context| set_node(rule, Value::Undefined));
1412 // Text and space concatenation.
1413 spec.add_open(with_push(alt(vec![val.clone(), vec![sp]], 2, ""), "text"));
1414 spec.add_open(with_push(alt(vec![vec![sp]], 1, ""), "text"));
1415 // A plain value token.
1416 spec.add_open(alt(vec![val.clone()], 0, ""));
1417 // LN ends the record.
1418 spec.add_open(alt(vec![vec![ln]], 1, ""));
1419 // Coalesce: a child (text) node wins; else the matched scalar
1420 // token; else no value (an implicit empty field).
1421 spec.add_bc(|rule, context| {
1422 let unset = rule.node.borrow().is_undefined();
1423 if unset {
1424 if !rule.child_node.is_undefined() {
1425 let child = rule.child_node.clone();
1426 set_node(rule, child);
1427 } else if rule.os() != 0 {
1428 let value = rule.resolve_open_value(0, context);
1429 set_node(rule, value);
1430 }
1431 }
1432 });
1433 });
1434}
1435
1436/// Non-strict mode: the CSV alternates go in FRONT of jsonic's, so the
1437/// default value alternates (handling `[1,2]`, `{x:1}` and the like) stay
1438/// available for an embedded value.
1439fn relaxed_rules(parser: &mut Tabnas, tokens: &Tokens, empty: &Value) {
1440 let Tokens {
1441 ln,
1442 ca,
1443 sp,
1444 zz,
1445 val,
1446 } = tokens;
1447 let (ln, ca, sp, zz) = (*ln, *ca, *sp, *zz);
1448
1449 parser.define_rule("list", |spec| {
1450 spec.add_bo(allocate_fields);
1451 // Do not consume the LN that closes an empty record.
1452 prepend_open(spec, vec![alt(vec![vec![ln]], 1, "")]);
1453 // The elem-push fallback goes after the jsonic defaults.
1454 spec.add_open(push_alt("elem"));
1455 // LN ends the record; in front so these win over jsonic's close
1456 // alternates.
1457 prepend_close(
1458 spec,
1459 vec![alt(vec![vec![ln]], 1, "end"), alt(vec![vec![zz]], 0, "end")],
1460 );
1461 });
1462
1463 let before_comma = empty.clone();
1464 let at_end = empty.clone();
1465 parser.define_rule("elem", move |spec| {
1466 let mut comma = alt(vec![vec![ca]], 1, "empty");
1467 comma.add_action(empty_before_comma(before_comma));
1468 prepend_open(spec, vec![comma]);
1469
1470 let mut trailing = alt(vec![vec![ca], vec![ln, zz]], 1, "comma,empty");
1471 trailing.add_action(empty_at_end(at_end));
1472 prepend_close(spec, vec![trailing, alt(vec![vec![ln]], 1, "end")]);
1473 });
1474
1475 let val = val.clone();
1476 parser.define_rule("val", move |spec| {
1477 prepend_open(
1478 spec,
1479 vec![
1480 with_push(alt(vec![val.clone(), vec![sp]], 2, ""), "text"),
1481 with_push(alt(vec![vec![sp]], 1, ""), "text"),
1482 alt(vec![vec![ln]], 1, ""),
1483 ],
1484 );
1485 });
1486}
1487
1488// ---------------------------------------------------------------------------
1489// Public surface
1490// ---------------------------------------------------------------------------
1491
1492/// Whether the CSV grammar is already on this instance. A plugin re-run
1493/// (a derived instance rebuilds its plugins on a fresh rule set; an
1494/// instance that already carries the grammar must not gain a second copy
1495/// of every alternate) is judged by the one rule only csv installs.
1496fn grammar_installed(parser: &Tabnas) -> bool {
1497 parser.rule_names().iter().any(|name| name == "csv")
1498}
1499
1500/// Install the CSV plugin on `parser`, which must already carry the jsonic
1501/// grammar: the counterpart of the TypeScript `Csv` plugin function and
1502/// the Go `Csv`.
1503///
1504/// ```
1505/// let mut parser = tabnas_jsonic::make();
1506/// tabnas_csv::csv(&mut parser, &tabnas_csv::CsvOptions::default())?;
1507/// let value = parser.parse("a,b\n1,2")?;
1508/// assert_eq!(value.to_json().to_string(), r#"[{"a":"1","b":"2"}]"#);
1509/// # Ok::<(), Box<dyn std::error::Error>>(())
1510/// ```
1511///
1512/// In order: the closures the grammar names are registered, the
1513/// per-instance engine options are applied, the embedded grammar is
1514/// installed with every alternate tagged `csv`, and the `list`, `elem` and
1515/// `val` rules are modified in code (replaced in strict mode, extended in
1516/// non-strict mode, so embedded jsonic values keep working).
1517pub fn csv(parser: &mut Tabnas, options: &CsvOptions) -> Result<(), PluginError> {
1518 if grammar_installed(parser) {
1519 return Ok(());
1520 }
1521
1522 let strict = options.strict;
1523
1524 // In strict mode jsonic value parsing is off and every field is raw
1525 // text. In non-strict mode the three-state options default to on.
1526 let (trim, comment, number, value) = if strict {
1527 (
1528 options.trim.unwrap_or(false),
1529 options.comment.unwrap_or(false),
1530 options.number.unwrap_or(false),
1531 options.value.unwrap_or(false),
1532 )
1533 } else {
1534 (
1535 options.trim.unwrap_or(true),
1536 options.comment.unwrap_or(true),
1537 options.number.unwrap_or(true),
1538 options.value.unwrap_or(true),
1539 )
1540 };
1541 let csv_strings = if strict {
1542 options.string.csv != Some(false)
1543 } else {
1544 options.string.csv == Some(true)
1545 };
1546
1547 let settings = Settings {
1548 strict,
1549 objres: options.object,
1550 header: options.header,
1551 trim,
1552 record_empty: options.record.empty,
1553 empty: Value::from_json(&options.field.empty),
1554 nonameprefix: options.field.nonameprefix.clone(),
1555 names: options.field.names.clone(),
1556 exact: options.field.exact,
1557 stream: options.stream.clone(),
1558 };
1559 let flags = LexFlags {
1560 number,
1561 value,
1562 comment,
1563 csv_strings,
1564 };
1565
1566 register_refs(parser, &settings, &options.string.quote);
1567 apply_document(parser, options_document(options, &settings, &flags))?;
1568
1569 let spec =
1570 GrammarSpec::from_value(grammar_document()?).map_err(|error| PluginError(error.0))?;
1571 parser
1572 .grammar_with_setting(&spec, &GrammarSetting::groups("csv"))
1573 .map_err(|error| PluginError(format!("csv: failed to apply the grammar: {}", error.0)))?;
1574
1575 // Usually [#TX, #ST, #NR, #VL].
1576 let tokens = Tokens {
1577 ln: parser.token("#LN"),
1578 ca: parser.token("#CA"),
1579 sp: parser.token("#SP"),
1580 zz: parser.token("#ZZ"),
1581 val: parser.token_set("VAL").unwrap_or_default(),
1582 };
1583 if strict {
1584 strict_rules(parser, &tokens, &settings.empty);
1585 } else {
1586 relaxed_rules(parser, &tokens, &settings.empty);
1587 }
1588 Ok(())
1589}
1590
1591/// The plugin form of [`csv`], for [`Tabnas::use_plugin`], with the
1592/// documented defaults: the counterpart of `tn.use(Csv, options)` in
1593/// TypeScript and `j.UseDefaults(Csv, Defaults, options)` in Go. The
1594/// option bag `use_plugin` takes is deep-merged over the defaults and
1595/// read as a [`CsvOptions`].
1596///
1597/// ```
1598/// use tabnas::Value;
1599///
1600/// let mut parser = tabnas_jsonic::make();
1601/// let options = Value::from_json(&serde_json::json!({ "object": false }));
1602/// parser.use_plugin(tabnas_csv::plugin(), Some(options))?;
1603/// assert_eq!(parser.parse("a,b\n1,2")?.to_json().to_string(), r#"[["1","2"]]"#);
1604/// # Ok::<(), Box<dyn std::error::Error>>(())
1605/// ```
1606pub fn plugin() -> Plugin {
1607 Plugin::new(PLUGIN_NAME, |parser, options| {
1608 let options = CsvOptions::from_value(options).map_err(PluginError)?;
1609 csv(parser, &options)
1610 })
1611 .with_defaults(CsvOptions::default().to_value())
1612}
1613
1614/// The plugin over typed options, including a [`Stream`] callback, which
1615/// cannot travel in the option bag. The bag `use_plugin` passes is
1616/// recorded on the instance as the plugin's options but the typed ones
1617/// are what the grammar is built from.
1618pub fn plugin_with(options: CsvOptions) -> Plugin {
1619 let defaults = options.to_value();
1620 Plugin::new(PLUGIN_NAME, move |parser, _bag| csv(parser, &options)).with_defaults(defaults)
1621}
1622
1623/// Build a CSV parser with the given options: the counterpart of
1624/// `new Tabnas().use(jsonic).use(Csv, options)` and the Go
1625/// `jsonic.Make()` + `UseDefaults(Csv, Defaults, options)`.
1626///
1627/// Infallible by design: the jsonic base and the embedded grammar are
1628/// fixed, so a failure here is a bug in this crate rather than anything a
1629/// caller did.
1630///
1631/// ```
1632/// use tabnas_csv::CsvOptions;
1633///
1634/// let parser = tabnas_csv::make_with(CsvOptions { header: false, object: false, ..Default::default() });
1635/// assert_eq!(parser.parse("a,b\n1,2")?.to_json().to_string(), r#"[["a","b"],["1","2"]]"#);
1636/// # Ok::<(), tabnas_csv::CsvError>(())
1637/// ```
1638pub fn make_with(options: CsvOptions) -> Tabnas {
1639 let mut parser = tabnas_jsonic::make();
1640 parser
1641 .use_plugin(plugin_with(options), None)
1642 .expect("the csv grammar documents are fixed and valid");
1643 // The quoted field, the one token this grammar cuts itself, reads the
1644 // source only through the lexer's window and fill, and jsonic's base
1645 // is the engine's own matchers, so a parse from a reader may hold a
1646 // window of the source rather than all of it (`Tabnas::parse_reader`,
1647 // admin ADR-30). Set on the parser this crate makes whole, not in
1648 // [`csv`].
1649 parser
1650 .set_options(|options| options.lex.window = true)
1651 .expect("lex.window sets");
1652 parser
1653}
1654
1655/// Build a CSV parser with the default options.
1656///
1657/// ```
1658/// let parser = tabnas_csv::make();
1659/// assert_eq!(parser.parse("a\n\"b\"\"c\"")?.to_json().to_string(), r#"[{"a":"b\"c"}]"#);
1660/// assert_eq!(parser.parse("a\n\"b").unwrap_err().code, "unterminated_string");
1661/// # Ok::<(), tabnas_csv::CsvError>(())
1662/// ```
1663pub fn make() -> Tabnas {
1664 make_with(CsvOptions::default())
1665}
1666
1667/// Parse a CSV source string with the shared default parser.
1668///
1669/// The engine is built once, on first use, and reused after that. Reuse
1670/// is safe: [`Tabnas::parse`] takes `&self` and builds a fresh parse
1671/// context per call, and `Tabnas` is `Send + Sync`, so concurrent callers
1672/// share one installed grammar instead of each rebuilding it, which is
1673/// what dominates a small parse.
1674///
1675/// Use [`make`] or [`make_with`] instead when the parser needs
1676/// configuring: that returns a fresh instance and leaves this one alone.
1677///
1678/// ```
1679/// let value = tabnas_csv::parse("a,b\n1,2\n3,4")?;
1680/// assert_eq!(value.to_json().to_string(), r#"[{"a":"1","b":"2"},{"a":"3","b":"4"}]"#);
1681/// # Ok::<(), tabnas_csv::CsvError>(())
1682/// ```
1683pub fn parse(src: &str) -> Result<Value, CsvError> {
1684 static DEFAULT: OnceLock<Tabnas> = OnceLock::new();
1685 DEFAULT.get_or_init(make).parse(src)
1686}
1687
1688/// One optional alchemy source and the entry point a host calls.
1689#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1690pub struct TranslationPart {
1691 /// The entry point a host calls.
1692 pub entry: &'static str,
1693 /// The source text, or `None` for an entry supplied by alchemy.
1694 pub source: Option<&'static str>,
1695}
1696
1697/// The package-local structural translation interface.
1698#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1699pub struct TranslationParts {
1700 /// The complete `tabnas.plugin.json` text.
1701 pub manifest: &'static str,
1702 /// An optional lift from the grammar's events to its first read shape.
1703 pub lift: Option<TranslationPart>,
1704 /// An optional embedding of a plain tree in the format's schema, with its reverse.
1705 pub embed: Option<TranslationPart>,
1706 /// An optional render from the write shape to text.
1707 pub render: Option<TranslationPart>,
1708 /// An optional transducer: the format's JSON events from its parser's
1709 /// rule passes, for a host that reads parser events.
1710 pub transduce: Option<TranslationPart>,
1711}
1712
1713const TRANSLATION: TranslationParts = TranslationParts {
1714 manifest: include_str!("../translate/manifest.json"),
1715 lift: None,
1716 embed: None,
1717 render: Some(TranslationPart {
1718 entry: "csv",
1719 source: None,
1720 }),
1721 transduce: Some(TranslationPart {
1722 entry: "csv-transduce",
1723 source: Some(include_str!("../translate/transduce.alc")),
1724 }),
1725};
1726
1727/// Return CSV's immutable translation parts.
1728#[must_use]
1729pub const fn translate() -> Option<TranslationParts> {
1730 Some(TRANSLATION)
1731}
1732
1733/// The plugin's manifest, `tabnas.plugin.json`, as the repository carries
1734/// it. Its `translate` object is what a host that translates reads: the
1735/// shapes CSV is read as (`tree`, one object per record, keyed by the
1736/// header) and written from (`records`), the root whose elements are the
1737/// rows (`array`), the render that writes it, which is the `csv` render
1738/// alchemy carries rather than a file of this repository's, and the
1739/// sentences that say what that render changes. Its `transduce` names the
1740/// transducer part, `alchemy/transduce.alc`, the document's JSON events
1741/// from the grammar's rule passes, through which a host that reads parser
1742/// events reads a document. The crate embeds its own copies,
1743/// `translate/manifest.json` and `translate/transduce.alc`, since a
1744/// packaged crate holds nothing outside `rs/`; `tests/translate_test.rs`
1745/// holds each copy to its file.
1746///
1747/// ```
1748/// assert!(tabnas_csv::manifest_text().contains("\"translate\""));
1749/// ```
1750pub fn manifest_text() -> &'static str {
1751 TRANSLATION.manifest
1752}