1use std::collections::HashMap;
30use std::fmt;
31
32pub mod plugins;
34
35#[cfg(feature = "serde")]
36use serde::{Deserialize, Serialize};
37
38pub const VERSION: &str = "1.0.2";
39
40#[derive(Debug, Clone)]
46pub struct ISONError {
47 pub message: String,
48 pub line: Option<usize>,
49}
50
51impl fmt::Display for ISONError {
52 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
53 match self.line {
54 Some(line) => write!(f, "Line {}: {}", line, self.message),
55 None => write!(f, "{}", self.message),
56 }
57 }
58}
59
60impl std::error::Error for ISONError {}
61
62pub type Result<T> = std::result::Result<T, ISONError>;
63
64#[derive(Debug, Clone, PartialEq)]
70#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
71pub struct Reference {
72 pub id: String,
73 pub ref_type: Option<String>,
74}
75
76impl Reference {
77 pub fn new(id: impl Into<String>) -> Self {
79 Self {
80 id: id.into(),
81 ref_type: None,
82 }
83 }
84
85 pub fn with_type(id: impl Into<String>, ref_type: impl Into<String>) -> Self {
87 Self {
88 id: id.into(),
89 ref_type: Some(ref_type.into()),
90 }
91 }
92
93 pub fn is_relationship(&self) -> bool {
95 match &self.ref_type {
96 Some(t) => t.chars().all(|c| c.is_uppercase() || c == '_'),
97 None => false,
98 }
99 }
100
101 pub fn get_namespace(&self) -> Option<&str> {
103 if self.is_relationship() {
104 None
105 } else {
106 self.ref_type.as_deref()
107 }
108 }
109
110 pub fn relationship_type(&self) -> Option<&str> {
112 if self.is_relationship() {
113 self.ref_type.as_deref()
114 } else {
115 None
116 }
117 }
118
119 pub fn to_ison(&self) -> String {
121 match &self.ref_type {
122 Some(t) => format!(":{}:{}", t, self.id),
123 None => format!(":{}", self.id),
124 }
125 }
126}
127
128impl fmt::Display for Reference {
129 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
130 write!(f, "{}", self.to_ison())
131 }
132}
133
134#[derive(Debug, Clone, PartialEq)]
136#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
137#[cfg_attr(feature = "serde", serde(untagged))]
138pub enum Value {
139 Null,
140 Bool(bool),
141 Int(i64),
142 Float(f64),
143 String(String),
144 Reference(Reference),
145}
146
147impl Value {
148 pub fn is_null(&self) -> bool {
149 matches!(self, Value::Null)
150 }
151
152 pub fn is_bool(&self) -> bool {
153 matches!(self, Value::Bool(_))
154 }
155
156 pub fn is_int(&self) -> bool {
157 matches!(self, Value::Int(_))
158 }
159
160 pub fn is_float(&self) -> bool {
161 matches!(self, Value::Float(_))
162 }
163
164 pub fn is_string(&self) -> bool {
165 matches!(self, Value::String(_))
166 }
167
168 pub fn is_reference(&self) -> bool {
169 matches!(self, Value::Reference(_))
170 }
171
172 pub fn as_bool(&self) -> Option<bool> {
173 match self {
174 Value::Bool(b) => Some(*b),
175 _ => None,
176 }
177 }
178
179 pub fn as_int(&self) -> Option<i64> {
180 match self {
181 Value::Int(i) => Some(*i),
182 _ => None,
183 }
184 }
185
186 pub fn as_float(&self) -> Option<f64> {
187 match self {
188 Value::Float(f) => Some(*f),
189 Value::Int(i) => Some(*i as f64),
190 _ => None,
191 }
192 }
193
194 pub fn as_str(&self) -> Option<&str> {
195 match self {
196 Value::String(s) => Some(s),
197 _ => None,
198 }
199 }
200
201 pub fn as_reference(&self) -> Option<&Reference> {
202 match self {
203 Value::Reference(r) => Some(r),
204 _ => None,
205 }
206 }
207}
208
209impl fmt::Display for Value {
210 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
211 match self {
212 Value::Null => write!(f, "null"),
213 Value::Bool(b) => write!(f, "{}", b),
214 Value::Int(i) => write!(f, "{}", i),
215 Value::Float(fl) => write!(f, "{}", fl),
216 Value::String(s) => write!(f, "{}", s),
217 Value::Reference(r) => write!(f, "{}", r),
218 }
219 }
220}
221
222pub type Row = HashMap<String, Value>;
224
225#[derive(Debug, Clone)]
227#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
228pub struct FieldInfo {
229 pub name: String,
230 pub field_type: Option<String>,
231 pub is_computed: bool,
232}
233
234impl FieldInfo {
235 pub fn new(name: impl Into<String>) -> Self {
236 Self {
237 name: name.into(),
238 field_type: None,
239 is_computed: false,
240 }
241 }
242
243 pub fn with_type(name: impl Into<String>, field_type: impl Into<String>) -> Self {
244 let ft: String = field_type.into();
245 let is_computed = ft == "computed";
246 Self {
247 name: name.into(),
248 field_type: Some(ft),
249 is_computed,
250 }
251 }
252}
253
254#[derive(Debug, Clone)]
256#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
257pub struct Block {
258 pub kind: String,
259 pub name: String,
260 pub fields: Vec<String>,
261 pub field_info: Vec<FieldInfo>,
262 pub rows: Vec<Row>,
263 pub summary_rows: Vec<Row>,
264}
265
266impl Block {
267 pub fn new(kind: impl Into<String>, name: impl Into<String>) -> Self {
268 Self {
269 kind: kind.into(),
270 name: name.into(),
271 fields: Vec::new(),
272 field_info: Vec::new(),
273 rows: Vec::new(),
274 summary_rows: Vec::new(),
275 }
276 }
277
278 pub fn len(&self) -> usize {
280 self.rows.len()
281 }
282
283 pub fn is_empty(&self) -> bool {
285 self.rows.is_empty()
286 }
287
288 pub fn get_row(&self, index: usize) -> Option<&Row> {
290 self.rows.get(index)
291 }
292
293 pub fn get_field_type(&self, field_name: &str) -> Option<&str> {
295 self.field_info
296 .iter()
297 .find(|fi| fi.name == field_name)
298 .and_then(|fi| fi.field_type.as_deref())
299 }
300
301 pub fn get_computed_fields(&self) -> Vec<&str> {
303 self.field_info
304 .iter()
305 .filter(|fi| fi.is_computed)
306 .map(|fi| fi.name.as_str())
307 .collect()
308 }
309}
310
311impl std::ops::Index<usize> for Block {
312 type Output = Row;
313
314 fn index(&self, index: usize) -> &Self::Output {
315 &self.rows[index]
316 }
317}
318
319#[derive(Debug, Clone, Default)]
321#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
322pub struct Document {
323 pub blocks: Vec<Block>,
324}
325
326impl Document {
327 pub fn new() -> Self {
328 Self { blocks: Vec::new() }
329 }
330
331 pub fn get(&self, name: &str) -> Option<&Block> {
333 self.blocks.iter().find(|b| b.name == name)
334 }
335
336 pub fn get_mut(&mut self, name: &str) -> Option<&mut Block> {
338 self.blocks.iter_mut().find(|b| b.name == name)
339 }
340
341 pub fn has(&self, name: &str) -> bool {
343 self.blocks.iter().any(|b| b.name == name)
344 }
345
346 pub fn len(&self) -> usize {
348 self.blocks.len()
349 }
350
351 pub fn is_empty(&self) -> bool {
353 self.blocks.is_empty()
354 }
355
356 #[cfg(feature = "serde")]
358 pub fn to_json(&self, pretty: bool) -> String {
359 let map: HashMap<&str, Vec<&Row>> = self
360 .blocks
361 .iter()
362 .map(|b| (b.name.as_str(), b.rows.iter().collect()))
363 .collect();
364
365 if pretty {
366 serde_json::to_string_pretty(&map).unwrap_or_default()
367 } else {
368 serde_json::to_string(&map).unwrap_or_default()
369 }
370 }
371}
372
373impl std::ops::Index<&str> for Document {
374 type Output = Block;
375
376 fn index(&self, name: &str) -> &Self::Output {
377 self.get(name).expect("Block not found")
378 }
379}
380
381struct Parser<'a> {
386 text: &'a str,
387 pos: usize,
388 line: usize,
389}
390
391impl<'a> Parser<'a> {
392 fn new(text: &'a str) -> Self {
393 Self {
394 text,
395 pos: 0,
396 line: 1,
397 }
398 }
399
400 fn parse(&mut self) -> Result<Document> {
401 let mut doc = Document::new();
402
403 self.skip_whitespace_and_comments();
404
405 while self.pos < self.text.len() {
406 if let Some(block) = self.parse_block()? {
407 doc.blocks.push(block);
408 }
409 self.skip_whitespace_and_comments();
410 }
411
412 Ok(doc)
413 }
414
415 fn parse_block(&mut self) -> Result<Option<Block>> {
416 let header_line = match self.read_line() {
417 Some(line) => line,
418 None => return Ok(None),
419 };
420
421 if header_line.starts_with('#') || header_line.is_empty() {
422 return Ok(None);
423 }
424
425 let dot_index = header_line.find('.').ok_or_else(|| ISONError {
426 message: format!("Invalid block header: {}", header_line),
427 line: Some(self.line),
428 })?;
429
430 let kind = header_line[..dot_index].trim().to_string();
431 let name = header_line[dot_index + 1..].trim().to_string();
432
433 if kind.is_empty() || name.is_empty() {
434 return Err(ISONError {
435 message: format!("Invalid block header: {}", header_line),
436 line: Some(self.line),
437 });
438 }
439
440 let mut block = Block::new(kind, name);
441
442 self.skip_empty_lines();
444 let fields_line = match self.read_line() {
445 Some(line) => line,
446 None => return Ok(Some(block)),
447 };
448
449 let field_tokens = self.tokenize_line(&fields_line);
450 for (token, _) in field_tokens {
451 if let Some(colon_idx) = token.find(':') {
452 let field_name = token[..colon_idx].to_string();
453 let field_type = token[colon_idx + 1..].to_string();
454 block.fields.push(field_name.clone());
455 block.field_info.push(FieldInfo::with_type(field_name, field_type));
456 } else {
457 block.fields.push(token.clone());
458 block.field_info.push(FieldInfo::new(token));
459 }
460 }
461
462 let mut in_summary = false;
464 while self.pos < self.text.len() {
465 let line = match self.peek_line() {
466 Some(line) => line,
467 None => break,
468 };
469
470 if line.is_empty() || Self::looks_like_header(&line) {
472 break;
473 }
474
475 self.read_line(); if line.starts_with('#') {
479 continue;
480 }
481
482 if line.trim() == "---" {
484 in_summary = true;
485 continue;
486 }
487
488 let mut values = self.tokenize_line(&line);
489
490 let keep = strip_inline_comment(&values);
492 values.truncate(keep);
493 if values.is_empty() {
494 continue;
495 }
496
497 check_extra_tokens(&values, block.fields.len(), Some(self.line.saturating_sub(1)))?;
499
500 let mut row = Row::new();
501 for (i, field) in block.fields.iter().enumerate() {
502 let value = match values.get(i) {
503 Some((token, true)) => Value::String(token.clone()),
505 Some((token, false)) => self.parse_value(token)?,
506 None => Value::Null,
508 };
509 row.insert(field.clone(), value);
510 }
511
512 if in_summary {
513 block.summary_rows.push(row);
514 } else {
515 block.rows.push(row);
516 }
517 }
518
519 Ok(Some(block))
520 }
521
522 fn tokenize_line(&self, line: &str) -> Vec<(String, bool)> {
529 let mut tokens = Vec::new();
530 let chars: Vec<char> = line.chars().collect();
531 let mut i = 0;
532
533 while i < chars.len() {
534 while i < chars.len() && (chars[i] == ' ' || chars[i] == '\t') {
536 i += 1;
537 }
538
539 if i >= chars.len() {
540 break;
541 }
542
543 if chars[i] == '"' {
545 let (token, new_pos) = self.parse_quoted_string(&chars, i);
546 tokens.push((token, true));
547 i = new_pos;
548 } else {
549 let start = i;
551 while i < chars.len() && chars[i] != ' ' && chars[i] != '\t' {
552 i += 1;
553 }
554 tokens.push((chars[start..i].iter().collect(), false));
555 }
556 }
557
558 tokens
559 }
560
561 fn looks_like_header(line: &str) -> bool {
566 fn is_identifier(s: &str) -> bool {
567 let mut chars = s.chars();
568 match chars.next() {
569 Some(c) if c.is_ascii_alphabetic() || c == '_' => {}
570 _ => return false,
571 }
572 chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
573 }
574
575 if line.split_whitespace().count() != 1 {
576 return false;
577 }
578 let parts: Vec<&str> = line.split('.').collect();
579 parts.len() == 2 && is_identifier(parts[0]) && is_identifier(parts[1])
580 }
581
582 fn parse_quoted_string(&self, chars: &[char], start: usize) -> (String, usize) {
583 let mut result = String::new();
584 let mut i = start + 1; while i < chars.len() {
587 if chars[i] == '\\' {
588 if i + 1 < chars.len() {
589 let next = chars[i + 1];
590 match next {
591 'n' => result.push('\n'),
592 't' => result.push('\t'),
593 'r' => result.push('\r'),
594 '\\' => result.push('\\'),
595 '"' => result.push('"'),
596 '|' => result.push('|'),
597 _ => result.push(next),
598 }
599 i += 2;
600 } else {
601 result.push('\\');
602 i += 1;
603 }
604 } else if chars[i] == '"' {
605 return (result, i + 1);
606 } else {
607 result.push(chars[i]);
608 i += 1;
609 }
610 }
611
612 (result, i)
613 }
614
615 fn parse_value(&self, token: &str) -> Result<Value> {
616 if token == "null" || token == "~" {
618 return Ok(Value::Null);
619 }
620
621 if token == "true" {
623 return Ok(Value::Bool(true));
624 }
625 if token == "false" {
626 return Ok(Value::Bool(false));
627 }
628
629 if token.starts_with(':') {
631 return self.parse_reference(token);
632 }
633
634 if let Ok(i) = token.parse::<i64>() {
636 return Ok(Value::Int(i));
637 }
638
639 if let Ok(f) = token.parse::<f64>() {
641 return Ok(Value::Float(f));
642 }
643
644 Ok(Value::String(token.to_string()))
646 }
647
648 fn parse_reference(&self, token: &str) -> Result<Value> {
649 let content = &token[1..]; let parts: Vec<&str> = content.split(':').collect();
651
652 match parts.len() {
653 1 => Ok(Value::Reference(Reference::new(parts[0]))),
654 2 => Ok(Value::Reference(Reference::with_type(parts[1], parts[0]))),
655 _ => Err(ISONError {
656 message: format!("Invalid reference: {}", token),
657 line: Some(self.line),
658 }),
659 }
660 }
661
662 fn read_line(&mut self) -> Option<String> {
663 if self.pos >= self.text.len() {
664 return None;
665 }
666
667 let start = self.pos;
668 while self.pos < self.text.len() && self.text.as_bytes()[self.pos] != b'\n' {
669 self.pos += 1;
670 }
671
672 let line = self.text[start..self.pos].trim().to_string();
673
674 if self.pos < self.text.len() {
675 self.pos += 1; }
677 self.line += 1;
678
679 Some(line)
680 }
681
682 fn peek_line(&self) -> Option<String> {
683 if self.pos >= self.text.len() {
684 return None;
685 }
686
687 let mut end = self.pos;
688 while end < self.text.len() && self.text.as_bytes()[end] != b'\n' {
689 end += 1;
690 }
691
692 Some(self.text[self.pos..end].trim().to_string())
693 }
694
695 fn skip_whitespace_and_comments(&mut self) {
696 while self.pos < self.text.len() {
697 let ch = self.text.as_bytes()[self.pos];
698 match ch {
699 b' ' | b'\t' | b'\r' => self.pos += 1,
700 b'\n' => {
701 self.pos += 1;
702 self.line += 1;
703 }
704 b'#' => {
705 while self.pos < self.text.len() && self.text.as_bytes()[self.pos] != b'\n' {
706 self.pos += 1;
707 }
708 }
709 _ => break,
710 }
711 }
712 }
713
714 fn skip_empty_lines(&mut self) {
715 while self.pos < self.text.len() {
716 let ch = self.text.as_bytes()[self.pos];
717 match ch {
718 b' ' | b'\t' | b'\r' => self.pos += 1,
719 b'\n' => {
720 self.pos += 1;
721 self.line += 1;
722 }
723 b'#' => {
724 while self.pos < self.text.len() && self.text.as_bytes()[self.pos] != b'\n' {
725 self.pos += 1;
726 }
727 }
728 _ => break,
729 }
730 }
731 }
732}
733
734fn strip_inline_comment(tokens: &[(String, bool)]) -> usize {
742 for (i, (token, was_quoted)) in tokens.iter().enumerate() {
743 if !was_quoted && token.starts_with('#') {
744 return i;
745 }
746 }
747 tokens.len()
748}
749
750fn check_extra_tokens(
753 tokens: &[(String, bool)],
754 field_count: usize,
755 line: Option<usize>,
756) -> Result<()> {
757 if tokens.len() <= field_count {
758 return Ok(());
759 }
760 Err(ISONError {
761 message: format!(
762 "Row has {} values but only {} fields (extra value: {:?})",
763 tokens.len(),
764 field_count,
765 tokens[field_count].0
766 ),
767 line,
768 })
769}
770
771fn is_header_part(s: &str) -> bool {
778 let mut chars = s.chars();
779 match chars.next() {
780 Some(c) if c.is_ascii_alphabetic() || c == '_' => {}
781 _ => return false,
782 }
783 chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
784}
785
786fn looks_like_block_header(s: &str) -> bool {
793 let parts: Vec<&str> = s.split('.').collect();
794 parts.len() == 2 && parts.iter().all(|p| is_header_part(p))
795}
796
797struct Serializer {
798 align_columns: bool,
799 delimiter: String,
800}
801
802impl Serializer {
803 fn new(align_columns: bool) -> Self {
804 Self { align_columns, delimiter: " ".to_string() }
805 }
806
807 fn with_delimiter(align_columns: bool, delimiter: &str) -> Self {
808 Self { align_columns, delimiter: delimiter.to_string() }
809 }
810
811 fn serialize(&self, doc: &Document) -> Result<String> {
812 let mut parts = Vec::with_capacity(doc.blocks.len());
813 for b in &doc.blocks {
814 validate_block_names(b)?;
815 validate_row_references(b, &REFERENCE_FORBIDDEN_ISON)?;
816 parts.push(self.serialize_block(b));
817 }
818 Ok(parts.join("\n\n"))
819 }
820
821 fn serialize_block(&self, block: &Block) -> String {
822 let mut lines = Vec::new();
823
824 lines.push(format!("{}.{}", block.kind, block.name));
826
827 let field_defs: Vec<String> = block
829 .field_info
830 .iter()
831 .map(|fi| {
832 if let Some(ref ft) = fi.field_type {
833 format!("{}:{}", fi.name, ft)
834 } else {
835 fi.name.clone()
836 }
837 })
838 .collect();
839 lines.push(field_defs.join(&self.delimiter));
840
841 let widths = if self.align_columns {
843 self.calculate_widths(block)
844 } else {
845 vec![]
846 };
847
848 for row in &block.rows {
850 lines.push(self.serialize_row(row, &block.fields, &widths));
851 }
852
853 if !block.summary_rows.is_empty() {
855 lines.push("---".to_string());
856 for row in &block.summary_rows {
857 lines.push(self.serialize_row(row, &block.fields, &widths));
858 }
859 }
860
861 lines.join("\n")
862 }
863
864 fn calculate_widths(&self, block: &Block) -> Vec<usize> {
865 let mut widths: Vec<usize> = block.fields.iter().map(|f| f.len()).collect();
866
867 for row in block.rows.iter().chain(block.summary_rows.iter()) {
868 for (i, field) in block.fields.iter().enumerate() {
869 if let Some(value) = row.get(field) {
870 let str_val = self.serialize_value(value);
871 if i < widths.len() {
872 widths[i] = widths[i].max(str_val.len());
873 }
874 }
875 }
876 }
877
878 widths
879 }
880
881 fn serialize_row(&self, row: &Row, fields: &[String], widths: &[usize]) -> String {
882 let mut values = Vec::new();
883
884 for (i, field) in fields.iter().enumerate() {
885 let value = row.get(field).cloned().unwrap_or(Value::Null);
886 let mut str_val = self.serialize_value(&value);
887
888 if self.align_columns && !widths.is_empty() && i < fields.len() - 1 {
889 while str_val.len() < widths[i] {
890 str_val.push(' ');
891 }
892 }
893 values.push(str_val);
894 }
895
896 values.join(&self.delimiter)
897 }
898
899 fn serialize_value(&self, value: &Value) -> String {
900 match value {
901 Value::Null => "null".to_string(),
902 Value::Bool(b) => if *b { "true" } else { "false" }.to_string(),
903 Value::Int(i) => i.to_string(),
904 Value::Float(f) => format_float(*f),
905 Value::Reference(r) => r.to_ison(),
906 Value::String(s) => self.serialize_string(s),
907 }
908 }
909
910 fn serialize_string(&self, s: &str) -> String {
911 if s.is_empty() {
912 return "\"\"".to_string();
913 }
914
915 let needs_quotes = s.contains(' ')
919 || s.contains('\t')
920 || s.contains('\n')
921 || s.contains('\r')
922 || s.contains('"')
923 || s.contains('\\')
924 || looks_like_block_header(s)
929 || s == "true"
930 || s == "false"
931 || s == "null"
932 || s == "~"
933 || s == "~"
934 || s.starts_with('#')
935 || s.starts_with(':')
936 || s.parse::<f64>().is_ok();
937
938 if !needs_quotes {
939 return s.to_string();
940 }
941
942 let escaped = s
943 .replace('\\', "\\\\")
944 .replace('"', "\\\"")
945 .replace('\n', "\\n")
946 .replace('\t', "\\t")
947 .replace('\r', "\\r");
948
949 format!("\"{}\"", escaped)
950 }
951}
952
953struct CanonicalSerializer;
958
959impl CanonicalSerializer {
960 fn new() -> Self {
961 Self
962 }
963
964 fn serialize(&self, doc: &Document) -> Result<String> {
965 for b in &doc.blocks {
966 validate_block_names(b)?;
967 validate_row_references(b, &REFERENCE_FORBIDDEN_ISON)?;
968 }
969
970 let mut sorted_blocks = doc.blocks.clone();
972 sorted_blocks.sort_by(|a, b| {
973 let key_a = format!("{}.{}", a.kind, a.name);
974 let key_b = format!("{}.{}", b.kind, b.name);
975 key_a.cmp(&key_b)
976 });
977
978 let parts: Vec<String> = sorted_blocks
979 .iter()
980 .map(|b| self.serialize_block_canonical(b))
981 .collect();
982 Ok(parts.join("\n\n"))
983 }
984
985 fn sort_fields_canonical(&self, fields: &[String]) -> Vec<String> {
986 let id_fields: Vec<String> = fields.iter().filter(|f| *f == "id").cloned().collect();
998 let mut other_fields: Vec<String> = fields.iter().filter(|f| *f != "id").cloned().collect();
999
1000 other_fields.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
1003
1004 [id_fields, other_fields].concat()
1006 }
1007
1008 fn sort_rows_by_key_canonical(&self, block: &Block, sorted_fields: &[String]) -> Vec<Row> {
1020 if block.rows.is_empty() || sorted_fields.is_empty() {
1021 return block.rows.clone();
1022 }
1023
1024 let mut sorted_rows = block.rows.clone();
1025
1026 sorted_rows.sort_by(|a, b| {
1027 for field in sorted_fields {
1028 let val_a = a.get(field);
1029 let val_b = b.get(field);
1030
1031 let is_null_a = val_a.is_none() || matches!(val_a, Some(Value::Null));
1033 let is_null_b = val_b.is_none() || matches!(val_b, Some(Value::Null));
1034
1035 let ord = match (is_null_a, is_null_b) {
1036 (true, true) => continue,
1037 (true, false) => std::cmp::Ordering::Greater,
1038 (false, true) => std::cmp::Ordering::Less,
1039 (false, false) => {
1040 let str_a = self.value_to_string(val_a.unwrap());
1041 let str_b = self.value_to_string(val_b.unwrap());
1042 str_a.as_bytes().cmp(str_b.as_bytes())
1043 }
1044 };
1045
1046 if ord != std::cmp::Ordering::Equal {
1047 return ord;
1048 }
1049 }
1050 std::cmp::Ordering::Equal
1051 });
1052
1053 sorted_rows
1054 }
1055
1056 fn serialize_isonl(&self, doc: &Document) -> Result<String> {
1057 let mut lines = Vec::new();
1058
1059 let mut sorted_blocks = doc.blocks.clone();
1061 sorted_blocks.sort_by(|a, b| {
1062 let key_a = format!("{}.{}", a.kind, a.name);
1063 let key_b = format!("{}.{}", b.kind, b.name);
1064 key_a.cmp(&key_b)
1065 });
1066
1067 for block in &sorted_blocks {
1068 validate_isonl_envelope(block)?;
1069 let header = format!("{}.{}", block.kind, block.name);
1070
1071 let sorted_fields = self.sort_fields_canonical(&block.fields);
1073
1074 let fields: Vec<String> = if !block.field_info.is_empty() {
1076 sorted_fields
1077 .iter()
1078 .map(|field_name| {
1079 if let Some(fi) = block.field_info.iter().find(|fi| &fi.name == field_name) {
1081 if let Some(ref ft) = fi.field_type {
1082 format!("{}:{}", fi.name, ft)
1083 } else {
1084 fi.name.clone()
1085 }
1086 } else {
1087 field_name.clone()
1088 }
1089 })
1090 .collect()
1091 } else {
1092 sorted_fields.clone()
1093 };
1094 let fields_str = fields.join(" ");
1095
1096 let sorted_rows = self.sort_rows_by_key_canonical(block, &sorted_fields);
1098
1099 for row in &sorted_rows {
1103 let values: Vec<String> = sorted_fields
1104 .iter()
1105 .map(|f| {
1106 row.get(f)
1107 .map(|v| self.serialize_value_canonical_isonl(v))
1108 .unwrap_or_else(|| "null".to_string())
1109 })
1110 .collect();
1111 lines.push(format!("{}|{}|{}", header, fields_str, values.join(" ")));
1112 }
1113 }
1114
1115 Ok(lines.join("\n"))
1116 }
1117
1118 fn serialize_block_canonical(&self, block: &Block) -> String {
1119 let mut lines = Vec::new();
1120
1121 lines.push(format!("{}.{}", block.kind, block.name));
1123
1124 let sorted_fields = self.sort_fields_canonical(&block.fields);
1126
1127 let field_defs: Vec<String> = if !block.field_info.is_empty() {
1129 sorted_fields
1130 .iter()
1131 .map(|field_name| {
1132 if let Some(fi) = block.field_info.iter().find(|fi| &fi.name == field_name) {
1134 if let Some(ref ft) = fi.field_type {
1135 format!("{}:{}", fi.name, ft)
1136 } else {
1137 fi.name.clone()
1138 }
1139 } else {
1140 field_name.clone()
1141 }
1142 })
1143 .collect()
1144 } else {
1145 sorted_fields.clone()
1146 };
1147 lines.push(field_defs.join(" "));
1148
1149 let sorted_rows = self.sort_rows_by_key_canonical(block, &sorted_fields);
1151
1152 for row in &sorted_rows {
1154 let values: Vec<String> = sorted_fields
1155 .iter()
1156 .map(|f| {
1157 row.get(f)
1158 .map(|v| self.serialize_value_canonical(v))
1159 .unwrap_or_else(|| "null".to_string())
1160 })
1161 .collect();
1162 lines.push(values.join(" "));
1163 }
1164
1165 if !block.summary_rows.is_empty() {
1167 lines.push("---".to_string());
1168 for row in &block.summary_rows {
1169 let values: Vec<String> = sorted_fields
1170 .iter()
1171 .map(|f| {
1172 row.get(f)
1173 .map(|v| self.serialize_value_canonical(v))
1174 .unwrap_or_else(|| "null".to_string())
1175 })
1176 .collect();
1177 lines.push(values.join(" "));
1178 }
1179 }
1180
1181 lines.join("\n")
1182 }
1183
1184 fn serialize_value_canonical(&self, value: &Value) -> String {
1185 match value {
1186 Value::Null => "null".to_string(),
1187 Value::Bool(b) => if *b { "true" } else { "false" }.to_string(),
1188 Value::Int(i) => i.to_string(),
1189 Value::Float(f) => format_float(*f),
1190 Value::Reference(r) => r.to_ison(),
1191 Value::String(s) => self.serialize_string_canonical(s),
1192 }
1193 }
1194
1195 fn serialize_value_canonical_isonl(&self, value: &Value) -> String {
1196 match value {
1197 Value::String(s) => self.serialize_string_canonical_isonl(s),
1198 other => self.serialize_value_canonical(other),
1199 }
1200 }
1201
1202 fn serialize_string_canonical_isonl(&self, s: &str) -> String {
1209 if s.is_empty() {
1210 return "\"\"".to_string();
1211 }
1212
1213 let needs_quotes = s.contains(' ')
1214 || s.contains('\t')
1215 || s.contains('\n')
1216 || s.contains('\r')
1217 || s.contains('"')
1218 || s.contains('\\')
1219 || s.contains('|')
1220 || s == "true"
1221 || s == "false"
1222 || s == "null"
1223 || s == "~"
1224 || s == "~"
1225 || s.starts_with('#')
1226 || s.starts_with(':')
1227 || s.parse::<f64>().is_ok();
1228
1229 if !needs_quotes {
1230 return s.to_string();
1231 }
1232
1233 format!(
1234 "\"{}\"",
1235 s.replace('\\', "\\\\")
1236 .replace('"', "\\\"")
1237 .replace('\n', "\\n")
1238 .replace('\t', "\\t")
1239 .replace('\r', "\\r")
1240 .replace('|', "\\|")
1241 )
1242 }
1243
1244 fn serialize_string_canonical(&self, s: &str) -> String {
1245 if s.is_empty() {
1246 return "\"\"".to_string();
1247 }
1248
1249 let needs_quotes = s.contains(' ')
1251 || s.contains('\t')
1252 || s.contains('\n')
1253 || s.contains('\r')
1254 || s.contains('"')
1255 || s.contains('\\')
1256 || looks_like_block_header(s)
1257 || s == "true"
1258 || s == "false"
1259 || s == "null"
1260 || s == "~"
1261 || s == "~"
1262 || s.starts_with('#')
1263 || s.starts_with(':')
1264 || s.parse::<f64>().is_ok();
1265
1266 if !needs_quotes {
1267 return s.to_string();
1268 }
1269
1270 let escaped = s
1271 .replace('\\', "\\\\")
1272 .replace('"', "\\\"")
1273 .replace('\n', "\\n")
1274 .replace('\t', "\\t")
1275 .replace('\r', "\\r");
1276
1277 format!("\"{}\"", escaped)
1278 }
1279
1280 fn value_to_string(&self, value: &Value) -> String {
1282 match value {
1283 Value::Null => String::new(), Value::Bool(b) => if *b { "true" } else { "false" }.to_string(),
1285 Value::Int(i) => i.to_string(),
1286 Value::Float(f) => format_float(*f),
1287 Value::Reference(r) => r.to_ison(),
1288 Value::String(s) => s.clone(),
1289 }
1290 }
1291}
1292
1293const ISONL_ENVELOPE_FORBIDDEN: &[char] = &['|', '"', '\\', ' ', '\t', '\n', '\r'];
1300
1301fn split_isonl_sections(line: &str) -> Vec<String> {
1308 let chars: Vec<char> = line.chars().collect();
1309 let mut sections = Vec::new();
1310 let mut current = String::new();
1311 let mut in_quotes = false;
1312 let mut i = 0;
1313
1314 while i < chars.len() {
1315 let ch = chars[i];
1316
1317 if in_quotes && ch == '\\' && i + 1 < chars.len() {
1318 current.push(ch);
1321 current.push(chars[i + 1]);
1322 i += 2;
1323 continue;
1324 }
1325
1326 if ch == '"' {
1327 in_quotes = !in_quotes;
1328 current.push(ch);
1329 } else if ch == '|' && !in_quotes {
1330 sections.push(current.trim().to_string());
1331 current = String::new();
1332 } else {
1333 current.push(ch);
1334 }
1335
1336 i += 1;
1337 }
1338
1339 sections.push(current.trim().to_string());
1340 sections
1341}
1342
1343fn tokenize_isonl_values(line: &str) -> Vec<(String, bool)> {
1351 let chars: Vec<char> = line.chars().collect();
1352 let mut tokens = Vec::new();
1353 let mut i = 0;
1354
1355 while i < chars.len() {
1356 while i < chars.len() && (chars[i] == ' ' || chars[i] == '\t') {
1358 i += 1;
1359 }
1360 if i >= chars.len() {
1361 break;
1362 }
1363
1364 if chars[i] == '"' {
1365 let mut result = String::new();
1367 i += 1; while i < chars.len() {
1369 let ch = chars[i];
1370 if ch == '"' {
1371 i += 1; break;
1373 }
1374 if ch == '\\' {
1375 if i + 1 < chars.len() {
1376 let next = chars[i + 1];
1377 match next {
1378 'n' => result.push('\n'),
1379 't' => result.push('\t'),
1380 'r' => result.push('\r'),
1381 '\\' => result.push('\\'),
1382 '"' => result.push('"'),
1383 '|' => result.push('|'),
1384 _ => result.push(next),
1385 }
1386 i += 2;
1387 } else {
1388 result.push('\\');
1389 i += 1;
1390 }
1391 } else {
1392 result.push(ch);
1393 i += 1;
1394 }
1395 }
1396 tokens.push((result, true));
1397 } else {
1398 let start = i;
1400 while i < chars.len() && chars[i] != ' ' && chars[i] != '\t' {
1401 i += 1;
1402 }
1403 tokens.push((chars[start..i].iter().collect(), false));
1404 }
1405 }
1406
1407 tokens
1408}
1409
1410fn isonl_quote_if_needed(s: &str) -> String {
1412 if s.is_empty() {
1413 return "\"\"".to_string();
1414 }
1415
1416 let needs_quote = s.contains(' ')
1417 || s.contains('\t')
1418 || s.contains('"')
1419 || s.contains('\n')
1420 || s.contains('\r')
1421 || s.contains('\\')
1422 || s.contains('|')
1423 || s == "true"
1424 || s == "false"
1425 || s == "null"
1426 || s == "~"
1427 || s.starts_with('#')
1428 || s.starts_with(':')
1429 || s.parse::<f64>().is_ok();
1430
1431 if !needs_quote {
1432 return s.to_string();
1433 }
1434
1435 let escaped = s
1436 .replace('\\', "\\\\")
1437 .replace('"', "\\\"")
1438 .replace('\n', "\\n")
1439 .replace('\t', "\\t")
1440 .replace('\r', "\\r")
1441 .replace('|', "\\|");
1442
1443 format!("\"{}\"", escaped)
1444}
1445
1446fn isonl_serialize_value(value: &Value) -> String {
1448 match value {
1449 Value::Null => "null".to_string(),
1450 Value::Bool(b) => if *b { "true" } else { "false" }.to_string(),
1451 Value::Int(i) => i.to_string(),
1452 Value::Float(f) => format_float(*f),
1453 Value::Reference(r) => r.to_ison(),
1454 Value::String(s) => isonl_quote_if_needed(s),
1455 }
1456}
1457
1458fn format_float(f: f64) -> String {
1470 if f == 0.0 && f.is_sign_negative() {
1471 return "-0.0".to_string();
1472 }
1473 let s = f.to_string();
1474 if !s.contains(['.', 'e', 'E', 'N', 'i']) {
1479 return format!("{}.0", s);
1480 }
1481 s
1482}
1483
1484const NAME_FORBIDDEN: [char; 4] = [' ', '\t', '\n', '\r'];
1504const FIELD_FORBIDDEN: [char; 6] = [' ', '\t', '\n', '\r', ':', '|'];
1505
1506fn describe_char(c: char) -> String {
1508 match c {
1509 ' ' => "a space".to_string(),
1510 '\t' => "a tab".to_string(),
1511 '\n' => "a newline".to_string(),
1512 '\r' => "a carriage return".to_string(),
1513 other => format!("'{}'", other),
1514 }
1515}
1516
1517fn validate_field_name(name: &str) -> Result<()> {
1519 if let Some(c) = name.chars().find(|c| FIELD_FORBIDDEN.contains(c)) {
1520 return Err(ISONError {
1521 message: format!(
1522 "field name '{}' contains {}, which has no unambiguous ISON encoding",
1523 name,
1524 describe_char(c)
1525 ),
1526 line: None,
1527 });
1528 }
1529 if name.starts_with('#') {
1530 return Err(ISONError {
1531 message: format!(
1532 "field name '{}' starts with '#', which begins a comment; '#' elsewhere in a name is fine",
1533 name
1534 ),
1535 line: None,
1536 });
1537 }
1538 if name.is_empty() {
1539 return Err(ISONError {
1540 message: "field name is empty".to_string(),
1541 line: None,
1542 });
1543 }
1544 Ok(())
1545}
1546
1547fn validate_block_name(kind: &str, name: &str) -> Result<()> {
1549 for (label, value) in [("kind", kind), ("name", name)] {
1550 if let Some(c) = value.chars().find(|c| NAME_FORBIDDEN.contains(c)) {
1551 return Err(ISONError {
1552 message: format!(
1553 "block {} '{}' contains {}, which has no unambiguous ISON encoding",
1554 label,
1555 value,
1556 describe_char(c)
1557 ),
1558 line: None,
1559 });
1560 }
1561 if value.is_empty() {
1562 return Err(ISONError {
1563 message: format!("block {} is empty", label),
1564 line: None,
1565 });
1566 }
1567 }
1568 if kind.contains('.') {
1571 return Err(ISONError {
1572 message: format!(
1573 "block kind '{}' contains '.', which separates kind from name",
1574 kind
1575 ),
1576 line: None,
1577 });
1578 }
1579 Ok(())
1580}
1581
1582fn validate_block_names(block: &Block) -> Result<()> {
1584 validate_block_name(&block.kind, &block.name)?;
1585 for field in &block.fields {
1586 validate_field_name(field)?;
1587 }
1588 Ok(())
1589}
1590
1591const REFERENCE_FORBIDDEN_ISON: [char; 4] = [' ', '\t', '\n', '\r'];
1607const REFERENCE_FORBIDDEN_ISONL: [char; 5] = [' ', '\t', '\n', '\r', '|'];
1608
1609fn validate_reference(r: &Reference, forbidden: &[char]) -> Result<()> {
1611 let parts: [(&str, Option<&String>); 2] =
1612 [("id", Some(&r.id)), ("type", r.ref_type.as_ref())];
1613 for (label, value) in parts {
1614 let Some(value) = value else { continue };
1615 if let Some(c) = value.chars().find(|c| forbidden.contains(c)) {
1616 return Err(ISONError {
1617 message: format!(
1618 "reference {} '{}' contains {}; a reference is written as ':type:id' with no quoting, so it has no unambiguous ISON encoding",
1619 label,
1620 value,
1621 describe_char(c)
1622 ),
1623 line: None,
1624 });
1625 }
1626 }
1627 if r.id.is_empty() {
1628 return Err(ISONError {
1629 message: "reference id is empty".to_string(),
1630 line: None,
1631 });
1632 }
1633 Ok(())
1634}
1635
1636fn validate_row_references(block: &Block, forbidden: &[char]) -> Result<()> {
1638 for row in block.rows.iter().chain(block.summary_rows.iter()) {
1639 for value in row.values() {
1640 if let Value::Reference(r) = value {
1641 validate_reference(r, forbidden)?;
1642 }
1643 }
1644 }
1645 Ok(())
1646}
1647
1648fn validate_isonl_envelope(block: &Block) -> Result<()> {
1650 validate_block_names(block)?;
1657 validate_row_references(block, &REFERENCE_FORBIDDEN_ISONL)?;
1658
1659 for (label, value) in [("kind", &block.kind), ("name", &block.name)] {
1660 if value.is_empty() {
1661 return Err(ISONError {
1662 message: format!("ISONL block {} must be non-empty", label),
1663 line: None,
1664 });
1665 }
1666 if value.contains(ISONL_ENVELOPE_FORBIDDEN) {
1667 return Err(ISONError {
1668 message: format!(
1669 "ISONL block {} '{}' contains characters that cannot be serialized (pipe, quote, backslash, or whitespace)",
1670 label, value
1671 ),
1672 line: None,
1673 });
1674 }
1675 }
1676 if block.kind.contains('.') {
1677 return Err(ISONError {
1678 message: format!("ISONL block kind '{}' must not contain '.'", block.kind),
1679 line: None,
1680 });
1681 }
1682 if block.kind.starts_with('#') {
1683 return Err(ISONError {
1684 message: format!("ISONL block kind '{}' must not start with '#'", block.kind),
1685 line: None,
1686 });
1687 }
1688 for field in &block.fields {
1689 if field.is_empty() {
1690 return Err(ISONError {
1691 message: "ISONL field names must be non-empty".to_string(),
1692 line: None,
1693 });
1694 }
1695 if field.contains(ISONL_ENVELOPE_FORBIDDEN) {
1696 return Err(ISONError {
1697 message: format!(
1698 "ISONL field name '{}' contains characters that cannot be serialized (pipe, quote, backslash, or whitespace)",
1699 field
1700 ),
1701 line: None,
1702 });
1703 }
1704 }
1705 Ok(())
1706}
1707
1708pub fn parse_isonl(text: &str) -> Result<Document> {
1710 let mut doc = Document::new();
1711 let mut block_map: HashMap<String, usize> = HashMap::new();
1712
1713 for (line_num, line) in text.lines().enumerate() {
1714 let line = line.trim();
1715 if line.is_empty() || line.starts_with('#') {
1716 continue;
1717 }
1718
1719 let sections = split_isonl_sections(line);
1720 if sections.len() != 3 {
1721 return Err(ISONError {
1722 message: format!("Invalid ISONL line: {}", line),
1723 line: Some(line_num + 1),
1724 });
1725 }
1726
1727 let header = sections[0].as_str();
1728 let fields_part = sections[1].as_str();
1729 let values_part = sections[2].as_str();
1730
1731 let dot_index = header.find('.').ok_or_else(|| ISONError {
1732 message: format!("Invalid ISONL header: {}", header),
1733 line: Some(line_num + 1),
1734 })?;
1735
1736 let kind = &header[..dot_index];
1737 let name = &header[dot_index + 1..];
1738 let key = format!("{}.{}", kind, name);
1739
1740 let block_idx = if let Some(&idx) = block_map.get(&key) {
1741 idx
1742 } else {
1743 let mut block = Block::new(kind, name);
1744
1745 for f in fields_part.split_whitespace() {
1747 if let Some(colon_idx) = f.find(':') {
1748 let field_name = f[..colon_idx].to_string();
1749 let field_type = f[colon_idx + 1..].to_string();
1750 block.fields.push(field_name.clone());
1751 block.field_info.push(FieldInfo::with_type(field_name, field_type));
1752 } else {
1753 block.fields.push(f.to_string());
1754 block.field_info.push(FieldInfo::new(f));
1755 }
1756 }
1757
1758 let idx = doc.blocks.len();
1759 block_map.insert(key, idx);
1760 doc.blocks.push(block);
1761 idx
1762 };
1763
1764 let parser = Parser::new("");
1766 let mut values = tokenize_isonl_values(values_part);
1767
1768 let keep = strip_inline_comment(&values);
1770 values.truncate(keep);
1771
1772 let block = &doc.blocks[block_idx];
1773
1774 check_extra_tokens(&values, block.fields.len(), Some(line_num + 1))?;
1776
1777 let mut row = Row::new();
1778 for (i, field) in block.fields.iter().enumerate() {
1779 let value = match values.get(i) {
1780 Some((token, true)) => Value::String(token.clone()),
1781 Some((token, false)) => parser.parse_value(token)?,
1782 None => Value::Null,
1784 };
1785 row.insert(field.clone(), value);
1786 }
1787
1788 doc.blocks[block_idx].rows.push(row);
1789 }
1790
1791 Ok(doc)
1792}
1793
1794pub fn dumps_isonl(doc: &Document) -> Result<String> {
1800 let mut lines = Vec::new();
1801
1802 for block in &doc.blocks {
1803 validate_isonl_envelope(block)?;
1804 let header = format!("{}.{}", block.kind, block.name);
1805 let fields: Vec<String> = block
1806 .field_info
1807 .iter()
1808 .map(|fi| {
1809 if let Some(ref ft) = fi.field_type {
1810 format!("{}:{}", fi.name, ft)
1811 } else {
1812 fi.name.clone()
1813 }
1814 })
1815 .collect();
1816 let fields_str = fields.join(" ");
1817
1818 for row in &block.rows {
1819 let values: Vec<String> = block
1820 .fields
1821 .iter()
1822 .map(|f| {
1823 row.get(f)
1824 .map(isonl_serialize_value)
1825 .unwrap_or_else(|| "null".to_string())
1826 })
1827 .collect();
1828 lines.push(format!("{}|{}|{}", header, fields_str, values.join(" ")));
1829 }
1830 }
1831
1832 Ok(lines.join("\n"))
1833}
1834
1835pub fn parse(text: &str) -> Result<Document> {
1841 Parser::new(text).parse()
1842}
1843
1844pub fn loads(text: &str) -> Result<Document> {
1846 parse(text)
1847}
1848
1849pub fn dumps(doc: &Document, align_columns: bool) -> Result<String> {
1855 Serializer::new(align_columns).serialize(doc)
1856}
1857
1858pub fn dumps_with_delimiter(
1865 doc: &Document,
1866 align_columns: bool,
1867 delimiter: &str,
1868) -> Result<String> {
1869 Serializer::with_delimiter(align_columns, delimiter).serialize(doc)
1870}
1871
1872pub fn dumps_canonical(doc: &Document) -> Result<String> {
1886 CanonicalSerializer::new().serialize(doc)
1887}
1888
1889pub fn dumps_canonical_isonl(doc: &Document) -> Result<String> {
1902 CanonicalSerializer::new().serialize_isonl(doc)
1903}
1904
1905pub fn loads_isonl(text: &str) -> Result<Document> {
1907 parse_isonl(text)
1908}
1909
1910pub fn ison_to_isonl(ison_text: &str) -> Result<String> {
1912 let doc = parse(ison_text)?;
1913 dumps_isonl(&doc)
1914}
1915
1916pub fn isonl_to_ison(isonl_text: &str) -> Result<String> {
1918 let doc = parse_isonl(isonl_text)?;
1919 dumps(&doc, false)
1920}
1921
1922#[cfg(feature = "serde")]
1924#[derive(Debug, Clone)]
1925pub struct JsonToIsonOptions {
1926 pub flatten: bool,
1928 pub align_columns: bool,
1930}
1931
1932#[cfg(feature = "serde")]
1933impl Default for JsonToIsonOptions {
1934 fn default() -> Self {
1935 Self {
1936 flatten: true,
1937 align_columns: false,
1938 }
1939 }
1940}
1941
1942#[cfg(feature = "serde")]
1948pub fn json_to_ison(json_text: &str) -> Result<String> {
1949 json_to_ison_with_options(json_text, JsonToIsonOptions::default())
1950}
1951
1952#[cfg(feature = "serde")]
1954pub fn json_to_ison_with_options(json_text: &str, opts: JsonToIsonOptions) -> Result<String> {
1955 let json_value: serde_json::Value = serde_json::from_str(json_text)
1956 .map_err(|e| ISONError { message: format!("JSON parse error: {}", e), line: None })?;
1957
1958 let obj = json_value.as_object()
1959 .ok_or_else(|| ISONError { message: "JSON must be an object".to_string(), line: None })?;
1960
1961 let mut doc = Document::new();
1962 let mut extra_blocks: Vec<Block> = Vec::new();
1963 let mut ref_counter: i64 = 1;
1964
1965 fn is_nested_object(val: &serde_json::Value) -> bool {
1967 val.is_object()
1968 }
1969
1970 fn is_array_of_objects(val: &serde_json::Value) -> bool {
1972 if let Some(arr) = val.as_array() {
1973 !arr.is_empty() && arr[0].is_object()
1974 } else {
1975 false
1976 }
1977 }
1978
1979 fn is_array_of_primitives(val: &serde_json::Value) -> bool {
1981 if let Some(arr) = val.as_array() {
1982 arr.is_empty() || (!arr[0].is_object() && !arr[0].is_array())
1983 } else {
1984 false
1985 }
1986 }
1987
1988 fn is_array_of_arrays(val: &serde_json::Value) -> bool {
1990 if let Some(arr) = val.as_array() {
1991 !arr.is_empty() && arr[0].is_array()
1992 } else {
1993 false
1994 }
1995 }
1996
1997 fn json_to_value(val: &serde_json::Value) -> Value {
1999 match val {
2000 serde_json::Value::Null => Value::Null,
2001 serde_json::Value::Bool(b) => Value::Bool(*b),
2002 serde_json::Value::Number(n) => {
2003 if let Some(i) = n.as_i64() {
2004 Value::Int(i)
2005 } else if let Some(f) = n.as_f64() {
2006 Value::Float(f)
2007 } else {
2008 Value::String(n.to_string())
2009 }
2010 }
2011 serde_json::Value::String(s) => {
2012 if s.starts_with(':') {
2013 let parts: Vec<&str> = s[1..].splitn(2, ':').collect();
2014 if parts.len() == 2 {
2015 Value::Reference(Reference::with_type(parts[1], parts[0]))
2016 } else {
2017 Value::Reference(Reference::new(parts[0]))
2018 }
2019 } else {
2020 Value::String(s.clone())
2021 }
2022 }
2023 _ => Value::String(val.to_string()),
2024 }
2025 }
2026
2027 for (block_name, block_value) in obj {
2028 if let Some(arr) = block_value.as_array() {
2029 if is_array_of_arrays(block_value) {
2031 let max_cols = arr.iter()
2032 .filter_map(|r| r.as_array())
2033 .map(|a| a.len())
2034 .max()
2035 .unwrap_or(0);
2036
2037 let fields: Vec<String> = (1..=max_cols).map(|i| format!("col{}", i)).collect();
2038 let field_info: Vec<FieldInfo> = fields.iter()
2039 .map(|f| FieldInfo::new(f))
2040 .collect();
2041
2042 let mut rows = Vec::new();
2043 for item in arr {
2044 if let Some(inner_arr) = item.as_array() {
2045 let mut row = Row::new();
2046 for (i, field) in fields.iter().enumerate() {
2047 if i < inner_arr.len() {
2048 row.insert(field.clone(), json_to_value(&inner_arr[i]));
2049 } else {
2050 row.insert(field.clone(), Value::Null);
2051 }
2052 }
2053 rows.push(row);
2054 }
2055 }
2056
2057 doc.blocks.push(Block {
2058 kind: "table".to_string(),
2059 name: block_name.clone(),
2060 fields,
2061 field_info,
2062 rows,
2063 summary_rows: vec![],
2064 });
2065 continue;
2066 }
2067
2068 if arr.is_empty() {
2070 continue;
2071 }
2072
2073 if !arr[0].is_object() {
2074 let fields = vec!["value".to_string()];
2076 let field_info = vec![FieldInfo::new("value")];
2077 let rows: Vec<Row> = arr.iter()
2078 .map(|v| {
2079 let mut row = Row::new();
2080 row.insert("value".to_string(), json_to_value(v));
2081 row
2082 })
2083 .collect();
2084
2085 doc.blocks.push(Block {
2086 kind: "table".to_string(),
2087 name: block_name.clone(),
2088 fields,
2089 field_info,
2090 rows,
2091 summary_rows: vec![],
2092 });
2093 continue;
2094 }
2095
2096 let mut field_set: Vec<String> = Vec::new();
2098 let mut rows = Vec::new();
2099
2100 for item in arr {
2101 if let Some(item_obj) = item.as_object() {
2102 let row_id: i64 = if let Some(id_val) = item_obj.get("id") {
2104 id_val.as_i64().unwrap_or_else(|| {
2105 let id = ref_counter;
2106 ref_counter += 1;
2107 id
2108 })
2109 } else {
2110 let id = ref_counter;
2111 ref_counter += 1;
2112 id
2113 };
2114
2115 let parent_ref = Reference::new(row_id.to_string());
2116
2117 let mut row = Row::new();
2118 for (key, val) in item_obj {
2119 if opts.flatten && (is_nested_object(val) || is_array_of_objects(val) || is_array_of_primitives(val)) {
2120 let nested_name = format!("{}_{}", block_name, key);
2121
2122 if is_nested_object(val) {
2123 if let Some(nested_obj) = val.as_object() {
2125 let parent_id_field = format!("{}_id", block_name);
2126 let mut nested_fields = vec![parent_id_field.clone()];
2127 let mut nested_row = Row::new();
2128 nested_row.insert(parent_id_field.clone(), Value::Reference(parent_ref.clone()));
2129
2130 for (nk, nv) in nested_obj {
2131 if !is_nested_object(nv) && !is_array_of_objects(nv) && !is_array_of_primitives(nv) {
2132 nested_row.insert(nk.clone(), json_to_value(nv));
2133 if !nested_fields.contains(nk) {
2134 nested_fields.push(nk.clone());
2135 }
2136 }
2137 }
2138
2139 if nested_row.len() > 1 {
2141 if let Some(existing) = extra_blocks.iter_mut().find(|b| b.name == nested_name) {
2142 for f in &nested_fields {
2143 if !existing.fields.contains(f) {
2144 existing.fields.push(f.clone());
2145 existing.field_info.push(FieldInfo::new(f));
2146 }
2147 }
2148 existing.rows.push(nested_row);
2149 } else {
2150 let field_info = nested_fields.iter().map(|f| FieldInfo::new(f)).collect();
2151 extra_blocks.push(Block {
2152 kind: "table".to_string(),
2153 name: nested_name.clone(),
2154 fields: nested_fields,
2155 field_info,
2156 rows: vec![nested_row],
2157 summary_rows: vec![],
2158 });
2159 }
2160 }
2161 }
2162 } else if is_array_of_objects(val) {
2163 if let Some(arr) = val.as_array() {
2165 for arr_item in arr {
2166 if let Some(nested_obj) = arr_item.as_object() {
2167 let parent_id_field = format!("{}_id", block_name);
2168 let mut nested_fields = vec![parent_id_field.clone()];
2169 let mut nested_row = Row::new();
2170 nested_row.insert(parent_id_field.clone(), Value::Reference(parent_ref.clone()));
2171
2172 for (nk, nv) in nested_obj {
2173 if !is_nested_object(nv) && !is_array_of_objects(nv) && !is_array_of_primitives(nv) {
2174 nested_row.insert(nk.clone(), json_to_value(nv));
2175 if !nested_fields.contains(nk) {
2176 nested_fields.push(nk.clone());
2177 }
2178 }
2179 }
2180
2181 if nested_row.len() > 1 {
2182 if let Some(existing) = extra_blocks.iter_mut().find(|b| b.name == nested_name) {
2183 for f in &nested_fields {
2184 if !existing.fields.contains(f) {
2185 existing.fields.push(f.clone());
2186 existing.field_info.push(FieldInfo::new(f));
2187 }
2188 }
2189 existing.rows.push(nested_row);
2190 } else {
2191 let field_info = nested_fields.iter().map(|f| FieldInfo::new(f)).collect();
2192 extra_blocks.push(Block {
2193 kind: "table".to_string(),
2194 name: nested_name.clone(),
2195 fields: nested_fields,
2196 field_info,
2197 rows: vec![nested_row],
2198 summary_rows: vec![],
2199 });
2200 }
2201 }
2202 }
2203 }
2204 }
2205 } else if is_array_of_primitives(val) {
2206 if let Some(arr) = val.as_array() {
2208 let parent_id_field = format!("{}_id", block_name);
2209 for prim in arr {
2210 let mut nested_row = Row::new();
2211 nested_row.insert(parent_id_field.clone(), Value::Reference(parent_ref.clone()));
2212 nested_row.insert("value".to_string(), json_to_value(prim));
2213
2214 if let Some(existing) = extra_blocks.iter_mut().find(|b| b.name == nested_name) {
2215 existing.rows.push(nested_row);
2216 } else {
2217 let nested_fields = vec![parent_id_field.clone(), "value".to_string()];
2218 let field_info = nested_fields.iter().map(|f| FieldInfo::new(f)).collect();
2219 extra_blocks.push(Block {
2220 kind: "table".to_string(),
2221 name: nested_name.clone(),
2222 fields: nested_fields,
2223 field_info,
2224 rows: vec![nested_row],
2225 summary_rows: vec![],
2226 });
2227 }
2228 }
2229 }
2230 }
2231 } else {
2233 row.insert(key.clone(), json_to_value(val));
2234 if !field_set.contains(key) {
2235 field_set.push(key.clone());
2236 }
2237 }
2238 }
2239 rows.push(row);
2240 }
2241 }
2242
2243 let field_info: Vec<FieldInfo> = field_set.iter()
2244 .map(|f| FieldInfo::new(f))
2245 .collect();
2246
2247 doc.blocks.push(Block {
2248 kind: "table".to_string(),
2249 name: block_name.clone(),
2250 fields: field_set,
2251 field_info,
2252 rows,
2253 summary_rows: vec![],
2254 });
2255 } else if let Some(obj_value) = block_value.as_object() {
2256 let row_id = obj_value.get("id")
2258 .and_then(|v| v.as_i64())
2259 .map(|i| i.to_string())
2260 .unwrap_or_else(|| block_name.clone());
2261 let parent_ref = Reference::new(row_id);
2262
2263 let mut fields: Vec<String> = Vec::new();
2264 let mut row = Row::new();
2265
2266 for (key, val) in obj_value {
2267 if opts.flatten && (is_nested_object(val) || is_array_of_objects(val) || is_array_of_primitives(val)) {
2268 let nested_name = format!("{}_{}", block_name, key);
2270 let parent_id_field = format!("{}_id", block_name);
2271
2272 if is_nested_object(val) {
2273 if let Some(nested_obj) = val.as_object() {
2274 let mut nested_fields = vec![parent_id_field.clone()];
2275 let mut nested_row = Row::new();
2276 nested_row.insert(parent_id_field.clone(), Value::Reference(parent_ref.clone()));
2277
2278 for (nk, nv) in nested_obj {
2279 if !is_nested_object(nv) && !is_array_of_objects(nv) && !is_array_of_primitives(nv) {
2280 nested_row.insert(nk.clone(), json_to_value(nv));
2281 nested_fields.push(nk.clone());
2282 }
2283 }
2284
2285 if nested_row.len() > 1 {
2286 let field_info = nested_fields.iter().map(|f| FieldInfo::new(f)).collect();
2287 extra_blocks.push(Block {
2288 kind: "table".to_string(),
2289 name: nested_name,
2290 fields: nested_fields,
2291 field_info,
2292 rows: vec![nested_row],
2293 summary_rows: vec![],
2294 });
2295 }
2296 }
2297 } else if is_array_of_primitives(val) {
2298 if let Some(arr) = val.as_array() {
2299 let nested_fields = vec![parent_id_field.clone(), "value".to_string()];
2300 let field_info = nested_fields.iter().map(|f| FieldInfo::new(f)).collect();
2301 let nested_rows: Vec<Row> = arr.iter().map(|prim| {
2302 let mut r = Row::new();
2303 r.insert(parent_id_field.clone(), Value::Reference(parent_ref.clone()));
2304 r.insert("value".to_string(), json_to_value(prim));
2305 r
2306 }).collect();
2307
2308 extra_blocks.push(Block {
2309 kind: "table".to_string(),
2310 name: nested_name,
2311 fields: nested_fields,
2312 field_info,
2313 rows: nested_rows,
2314 summary_rows: vec![],
2315 });
2316 }
2317 }
2318 } else {
2319 row.insert(key.clone(), json_to_value(val));
2320 fields.push(key.clone());
2321 }
2322 }
2323
2324 let field_info: Vec<FieldInfo> = fields.iter()
2325 .map(|f| FieldInfo::new(f))
2326 .collect();
2327
2328 doc.blocks.push(Block {
2329 kind: "object".to_string(),
2330 name: block_name.clone(),
2331 fields,
2332 field_info,
2333 rows: vec![row],
2334 summary_rows: vec![],
2335 });
2336 }
2337 }
2338
2339 for block in extra_blocks {
2341 doc.blocks.push(block);
2342 }
2343
2344 dumps(&doc, opts.align_columns)
2345}
2346
2347#[cfg(feature = "serde")]
2354pub fn json_to_document(json_text: &str) -> Result<Document> {
2355 let json_value: serde_json::Value = serde_json::from_str(json_text)
2357 .map_err(|e| ISONError { message: format!("JSON parse error: {}", e), line: None })?;
2358
2359 let obj = json_value.as_object()
2360 .ok_or_else(|| ISONError { message: "JSON must be an object".to_string(), line: None })?;
2361
2362 let mut doc = Document::new();
2363 let mut extra_blocks: Vec<Block> = Vec::new();
2364
2365
2366 fn is_array_of_arrays(val: &serde_json::Value) -> bool {
2368 if let Some(arr) = val.as_array() {
2369 !arr.is_empty() && arr[0].is_array()
2370 } else {
2371 false
2372 }
2373 }
2374
2375 fn json_to_value(val: &serde_json::Value) -> Value {
2377 match val {
2378 serde_json::Value::Null => Value::Null,
2379 serde_json::Value::Bool(b) => Value::Bool(*b),
2380 serde_json::Value::Number(n) => {
2381 if let Some(i) = n.as_i64() {
2382 Value::Int(i)
2383 } else if let Some(f) = n.as_f64() {
2384 Value::Float(f)
2385 } else {
2386 Value::String(n.to_string())
2387 }
2388 }
2389 serde_json::Value::String(s) => {
2390 if s.starts_with(':') {
2391 let parts: Vec<&str> = s[1..].splitn(2, ':').collect();
2392 if parts.len() == 2 {
2393 Value::Reference(Reference::with_type(parts[1], parts[0]))
2394 } else {
2395 Value::Reference(Reference::new(parts[0]))
2396 }
2397 } else {
2398 Value::String(s.clone())
2399 }
2400 }
2401 _ => Value::String(val.to_string()),
2402 }
2403 }
2404
2405 for (block_name, block_value) in obj {
2406 if let Some(arr) = block_value.as_array() {
2407 if is_array_of_arrays(block_value) {
2409 let max_cols = arr.iter()
2410 .filter_map(|r| r.as_array())
2411 .map(|a| a.len())
2412 .max()
2413 .unwrap_or(0);
2414
2415 let fields: Vec<String> = (1..=max_cols).map(|i| format!("col{}", i)).collect();
2416 let field_info: Vec<FieldInfo> = fields.iter()
2417 .map(|f| FieldInfo::new(f))
2418 .collect();
2419
2420 let mut rows = Vec::new();
2421 for item in arr {
2422 if let Some(inner_arr) = item.as_array() {
2423 let mut row = Row::new();
2424 for (i, field) in fields.iter().enumerate() {
2425 if i < inner_arr.len() {
2426 row.insert(field.clone(), json_to_value(&inner_arr[i]));
2427 } else {
2428 row.insert(field.clone(), Value::Null);
2429 }
2430 }
2431 rows.push(row);
2432 }
2433 }
2434
2435 doc.blocks.push(Block {
2436 kind: "table".to_string(),
2437 name: block_name.clone(),
2438 fields,
2439 field_info,
2440 rows,
2441 summary_rows: vec![],
2442 });
2443 continue;
2444 }
2445
2446 if arr.is_empty() {
2448 continue;
2449 }
2450
2451 if !arr[0].is_object() {
2452 let fields = vec!["value".to_string()];
2454 let field_info = vec![FieldInfo::new("value")];
2455 let rows: Vec<Row> = arr.iter()
2456 .map(|v| {
2457 let mut row = Row::new();
2458 row.insert("value".to_string(), json_to_value(v));
2459 row
2460 })
2461 .collect();
2462
2463 doc.blocks.push(Block {
2464 kind: "table".to_string(),
2465 name: block_name.clone(),
2466 fields,
2467 field_info,
2468 rows,
2469 summary_rows: vec![],
2470 });
2471 continue;
2472 }
2473
2474 let mut field_set: Vec<String> = Vec::new();
2476 let mut rows = Vec::new();
2477
2478 for item in arr {
2479 if let Some(item_obj) = item.as_object() {
2480 let mut row = Row::new();
2481 for (key, val) in item_obj {
2482 row.insert(key.clone(), json_to_value(val));
2483 if !field_set.contains(key) {
2484 field_set.push(key.clone());
2485 }
2486 }
2487 rows.push(row);
2488 }
2489 }
2490
2491 let field_info: Vec<FieldInfo> = field_set.iter()
2492 .map(|f| FieldInfo::new(f))
2493 .collect();
2494
2495 doc.blocks.push(Block {
2496 kind: "table".to_string(),
2497 name: block_name.clone(),
2498 fields: field_set,
2499 field_info,
2500 rows,
2501 summary_rows: vec![],
2502 });
2503 } else if let Some(obj_value) = block_value.as_object() {
2504 let mut fields: Vec<String> = Vec::new();
2506 let mut row = Row::new();
2507
2508 for (key, val) in obj_value {
2509 row.insert(key.clone(), json_to_value(val));
2510 fields.push(key.clone());
2511 }
2512
2513 let field_info = fields.iter().map(|f| FieldInfo::new(f)).collect();
2514
2515 doc.blocks.push(Block {
2516 kind: "table".to_string(),
2517 name: block_name.clone(),
2518 fields,
2519 field_info,
2520 rows: vec![row],
2521 summary_rows: vec![],
2522 });
2523 }
2524 }
2525
2526 for block in extra_blocks {
2528 doc.blocks.push(block);
2529 }
2530
2531 Ok(doc)
2532}
2533
2534#[cfg(feature = "serde")]
2536pub fn json_to_ison_canonical(json_text: &str) -> Result<String> {
2537 dumps_canonical(&json_to_document(json_text)?)
2539}
2540
2541#[cfg(feature = "serde")]
2543pub fn ison_to_json(ison_text: &str, pretty: bool) -> Result<String> {
2544 let doc = parse(ison_text)?;
2545 Ok(doc.to_json(pretty))
2546}
2547
2548#[cfg(test)]
2549mod tests {
2550 use super::*;
2551
2552 #[test]
2553 fn test_parse_simple_table() {
2554 let ison = r#"table.users
2555id name email
25561 Alice alice@example.com
25572 Bob bob@example.com"#;
2558
2559 let doc = parse(ison).unwrap();
2560 let users = doc.get("users").unwrap();
2561
2562 assert_eq!(users.kind, "table");
2563 assert_eq!(users.name, "users");
2564 assert_eq!(users.len(), 2);
2565 assert_eq!(users.fields, vec!["id", "name", "email"]);
2566
2567 assert_eq!(users[0].get("id").unwrap().as_int(), Some(1));
2568 assert_eq!(users[0].get("name").unwrap().as_str(), Some("Alice"));
2569 }
2570
2571 #[test]
2572 fn test_parse_references() {
2573 let ison = r#"table.orders
2574id user_id
25751 :42
25762 :user:101
25773 :MEMBER_OF:10"#;
2578
2579 let doc = parse(ison).unwrap();
2580 let orders = doc.get("orders").unwrap();
2581
2582 let ref1 = orders[0].get("user_id").unwrap().as_reference().unwrap();
2583 assert_eq!(ref1.id, "42");
2584 assert!(ref1.ref_type.is_none());
2585
2586 let ref2 = orders[1].get("user_id").unwrap().as_reference().unwrap();
2587 assert_eq!(ref2.id, "101");
2588 assert_eq!(ref2.ref_type, Some("user".to_string()));
2589 assert!(!ref2.is_relationship());
2590
2591 let ref3 = orders[2].get("user_id").unwrap().as_reference().unwrap();
2592 assert_eq!(ref3.id, "10");
2593 assert!(ref3.is_relationship());
2594 }
2595
2596 #[test]
2597 fn test_type_inference() {
2598 let ison = r#"table.test
2599int_val float_val bool_val null_val str_val
260042 3.14 true null hello"#;
2601
2602 let doc = parse(ison).unwrap();
2603 let test = doc.get("test").unwrap();
2604
2605 assert!(test[0].get("int_val").unwrap().is_int());
2606 assert!(test[0].get("float_val").unwrap().is_float());
2607 assert!(test[0].get("bool_val").unwrap().is_bool());
2608 assert!(test[0].get("null_val").unwrap().is_null());
2609 assert!(test[0].get("str_val").unwrap().is_string());
2610 }
2611
2612 #[test]
2613 fn test_roundtrip() {
2614 let original = r#"table.users
2615id name email
26161 Alice alice@example.com
26172 Bob bob@example.com"#;
2618
2619 let doc = parse(original).unwrap();
2620 let serialized = dumps(&doc, true).unwrap();
2621 let doc2 = parse(&serialized).unwrap();
2622
2623 assert_eq!(doc2.get("users").unwrap().len(), 2);
2624 }
2625
2626 #[test]
2627 fn test_isonl() {
2628 let isonl = "table.users|id name|1 Alice\ntable.users|id name|2 Bob";
2629
2630 let doc = parse_isonl(isonl).unwrap();
2631 let users = doc.get("users").unwrap();
2632
2633 assert_eq!(users.len(), 2);
2634 assert_eq!(users[0].get("name").unwrap().as_str(), Some("Alice"));
2635 }
2636
2637 #[test]
2638 fn test_dumps_with_delimiter() {
2639 let ison = r#"table.users
2640id name email
26411 Alice "alice@example.com"
26422 Bob "bob@example.com""#;
2643
2644 let doc = parse(ison).unwrap();
2645
2646 let comma_output = dumps_with_delimiter(&doc, false, ",").unwrap();
2650 assert!(comma_output.contains("id,name,email"));
2651 assert!(comma_output.contains("1,Alice,alice@example.com"));
2652
2653 let space_output = dumps_with_delimiter(&doc, false, " ").unwrap();
2655 assert!(space_output.contains("id name email"));
2656 assert!(space_output.contains("1 Alice alice@example.com"));
2657 }
2658
2659 #[test]
2660 fn test_version() {
2661 assert_eq!(VERSION, "1.0.2");
2662 }
2663
2664 #[test]
2665 fn test_json_to_ison() {
2666 let json = r#"{
2667 "users": [
2668 {"id": 1, "name": "Alice", "email": "alice@example.com"},
2669 {"id": 2, "name": "Bob", "email": "bob@example.com"}
2670 ]
2671 }"#;
2672
2673 let ison = json_to_ison(json).unwrap();
2674 assert!(ison.contains("table.users"));
2675
2676 let doc = parse(&ison).unwrap();
2678 let users = doc.get("users").unwrap();
2679 assert_eq!(users.len(), 2);
2680 }
2681
2682 #[test]
2683 fn test_ison_to_json() {
2684 let ison = r#"table.users
2685id name email
26861 Alice alice@example.com
26872 Bob bob@example.com"#;
2688
2689 let json = ison_to_json(ison, false).unwrap();
2690 assert!(json.contains("Alice"));
2691 assert!(json.contains("Bob"));
2692
2693 let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
2695 assert!(parsed.get("users").is_some());
2696 }
2697
2698 fn make_string_block(kind: &str, name: &str, fields: &[&str], rows: Vec<Row>) -> Document {
2699 let mut block = Block::new(kind, name);
2700 for f in fields {
2701 block.fields.push(f.to_string());
2702 block.field_info.push(FieldInfo::new(*f));
2703 }
2704 block.rows = rows;
2705 let mut doc = Document::new();
2706 doc.blocks.push(block);
2707 doc
2708 }
2709
2710 #[test]
2711 fn test_isonl_escaping_integrity() {
2712 let adversarial: Vec<&str> = vec![
2714 "C:\\path\\", "\\",
2716 "a\\",
2717 "ends with backslash \\",
2718 "pipe|inside",
2719 "quote \" inside",
2720 "mix \\\" of both",
2721 "line1\nline2",
2722 "tab\there",
2723 "cr\rhere",
2724 "crlf\r\nend",
2725 "123",
2726 "true",
2727 ":ref",
2728 "",
2729 "\\|",
2730 " leading and trailing ",
2731 ];
2732
2733 let rows: Vec<Row> = adversarial
2734 .iter()
2735 .map(|s| {
2736 let mut row = Row::new();
2737 row.insert("v".to_string(), Value::String(s.to_string()));
2738 row
2739 })
2740 .collect();
2741 let doc = make_string_block("table", "adversarial", &["v"], rows);
2742
2743 let out = dumps_isonl(&doc).unwrap();
2744 let parsed = parse_isonl(&out).unwrap();
2745 let got: Vec<String> = parsed.blocks[0]
2746 .rows
2747 .iter()
2748 .map(|r| r.get("v").unwrap().as_str().unwrap().to_string())
2749 .collect();
2750 assert_eq!(got, adversarial, "round-trip corrupted values");
2751
2752 let compound_rows: Vec<Row> = [("x \\", "y|z"), ("x\\", "y|z")]
2756 .iter()
2757 .map(|(a, b)| {
2758 let mut row = Row::new();
2759 row.insert("a".to_string(), Value::String(a.to_string()));
2760 row.insert("b".to_string(), Value::String(b.to_string()));
2761 row
2762 })
2763 .collect();
2764 let doc = make_string_block("table", "compound", &["a", "b"], compound_rows.clone());
2765
2766 let out = dumps_isonl(&doc).unwrap();
2767 let parsed = parse_isonl(&out).unwrap();
2768 assert_eq!(parsed.blocks[0].rows, compound_rows);
2769 }
2770
2771 #[test]
2772 fn test_isonl_roundtrip_property() {
2773 struct Lcg {
2776 state: u64,
2777 }
2778 impl Lcg {
2779 fn next_int(&mut self, lo: u64, hi: u64) -> u64 {
2780 self.state = (self.state * 1103515245 + 12345) % 2147483648;
2781 lo + (self.state % (hi - lo + 1))
2782 }
2783 }
2784
2785 let alphabet: [&str; 16] = [
2786 "a", "b", " ", "|", "\"", "\\", "\n", "\r", "\t", ".", ":", "#", "0", "1", "true",
2787 "null",
2788 ];
2789 let mut rng = Lcg { state: 20260713 };
2790
2791 for trial in 0..300 {
2792 let num_fields = rng.next_int(1, 4) as usize;
2793 let fields: Vec<String> = (0..num_fields).map(|i| format!("f{}", i)).collect();
2794 let field_refs: Vec<&str> = fields.iter().map(|f| f.as_str()).collect();
2795
2796 let num_rows = rng.next_int(1, 3) as usize;
2797 let mut rows: Vec<Row> = Vec::new();
2798 for _ in 0..num_rows {
2799 let mut row = Row::new();
2800 for f in &fields {
2801 let len = rng.next_int(0, 12) as usize;
2802 let mut s = String::new();
2803 for _ in 0..len {
2804 s.push_str(alphabet[rng.next_int(0, alphabet.len() as u64 - 1) as usize]);
2805 }
2806 row.insert(f.clone(), Value::String(s));
2807 }
2808 rows.push(row);
2809 }
2810
2811 let doc = make_string_block("table", "t", &field_refs, rows.clone());
2812 let out = dumps_isonl(&doc).unwrap();
2813 let parsed = parse_isonl(&out)
2814 .unwrap_or_else(|e| panic!("trial {}: parse failed ({}) for {:?}", trial, e, out));
2815 assert_eq!(
2816 parsed.blocks[0].rows, rows,
2817 "trial {}: {:?} -> {:?}",
2818 trial, rows, out
2819 );
2820 }
2821 }
2822
2823 #[test]
2824 fn test_isonl_envelope_validation() {
2825 fn make_doc(kind: &str, name: &str, fields: &[&str]) -> Document {
2827 let mut row = Row::new();
2828 for f in fields {
2829 row.insert(f.to_string(), Value::Int(1));
2830 }
2831 make_string_block(kind, name, fields, vec![row])
2832 }
2833
2834 let bad_cases: Vec<Document> = vec![
2835 make_doc("ta|ble", "t", &["id"]),
2836 make_doc("ta ble", "t", &["id"]),
2837 make_doc("t.able", "t", &["id"]),
2838 make_doc("#table", "t", &["id"]),
2839 make_doc("", "t", &["id"]),
2840 make_doc("table", "na|me", &["id"]),
2841 make_doc("table", "na\nme", &["id"]),
2842 make_doc("table", "na me", &["id"]),
2843 make_doc("table", "", &["id"]),
2844 make_doc("table", "t", &["bad field"]),
2845 make_doc("table", "t", &["bad|field"]),
2846 make_doc("table", "t", &[""]),
2847 ];
2848 for doc in &bad_cases {
2849 assert!(
2850 dumps_isonl(doc).is_err(),
2851 "should have rejected envelope kind={:?} name={:?} fields={:?}",
2852 doc.blocks[0].kind,
2853 doc.blocks[0].name,
2854 doc.blocks[0].fields
2855 );
2856 }
2857
2858 let doc = make_doc("table", "v1.2", &["id"]);
2860 let parsed = parse_isonl(&dumps_isonl(&doc).unwrap()).unwrap();
2861 assert_eq!(parsed.blocks[0].kind, "table");
2862 assert_eq!(parsed.blocks[0].name, "v1.2");
2863 }
2864
2865 #[test]
2866 fn test_extra_values_rejected() {
2867 let err = parse("table.t\na b\n1 2 3").unwrap_err();
2869 assert!(
2870 err.to_string().contains("3 values"),
2871 "unexpected error message: {}",
2872 err
2873 );
2874
2875 assert!(parse("table.t\na b\n1 2 \"#not-a-comment\"").is_err());
2877
2878 let err = parse_isonl("table.t|a b|1 2 3").unwrap_err();
2880 assert!(
2881 err.to_string().contains("3 values"),
2882 "unexpected error message: {}",
2883 err
2884 );
2885 }
2886
2887 #[test]
2888 fn test_inline_trailing_comment() {
2889 let doc = parse("table.t\na b\n1 2 # note ignored").unwrap();
2891 let row = &doc.blocks[0].rows[0];
2892 assert_eq!(row.get("a"), Some(&Value::Int(1)));
2893 assert_eq!(row.get("b"), Some(&Value::Int(2)));
2894 assert_eq!(row.len(), 2);
2895
2896 let doc = parse_isonl("table.t|a b|1 2 # note ignored").unwrap();
2897 let row = &doc.blocks[0].rows[0];
2898 assert_eq!(row.get("a"), Some(&Value::Int(1)));
2899 assert_eq!(row.get("b"), Some(&Value::Int(2)));
2900 assert_eq!(row.len(), 2);
2901
2902 let doc = parse("table.t\na b\n1 #tag").unwrap();
2904 let row = &doc.blocks[0].rows[0];
2905 assert_eq!(row.get("a"), Some(&Value::Int(1)));
2906 assert_eq!(row.get("b"), Some(&Value::Null));
2907
2908 let doc = parse("table.t\na b\n1 \"#tag\"").unwrap();
2910 assert_eq!(
2911 doc.blocks[0].rows[0].get("b"),
2912 Some(&Value::String("#tag".to_string()))
2913 );
2914
2915 let mut row = Row::new();
2917 row.insert("a".to_string(), Value::String("#tag".to_string()));
2918 let doc = make_string_block("table", "t", &["a"], vec![row.clone()]);
2919 let parsed = parse(&dumps(&doc, false).unwrap()).unwrap();
2920 assert_eq!(parsed.blocks[0].rows, vec![row]);
2921
2922 let doc = parse("table.t\na b\n1 \"a#b\"").unwrap();
2924 assert_eq!(
2925 doc.blocks[0].rows[0].get("b"),
2926 Some(&Value::String("a#b".to_string()))
2927 );
2928
2929 let mut row = Row::new();
2933 row.insert("a".to_string(), Value::String("x\\".to_string()));
2934 row.insert("b".to_string(), Value::String("a #b".to_string()));
2935 let doc = make_string_block("table", "t", &["a", "b"], vec![row.clone()]);
2936 let out = dumps(&doc, false).unwrap();
2937 let parsed = parse(&out)
2938 .unwrap_or_else(|e| panic!("round-trip parse failed ({}) for {:?}", e, out));
2939 assert_eq!(parsed.blocks[0].rows, vec![row], "corrupted by {:?}", out);
2940 }
2941
2942 #[test]
2943 fn test_ison_roundtrip_property() {
2944 let header_shaped: Vec<Row> = ["a.true", "object.config"]
2949 .iter()
2950 .map(|s| {
2951 let mut row = Row::new();
2952 row.insert("v".to_string(), Value::String(s.to_string()));
2953 row
2954 })
2955 .collect();
2956 let doc = make_string_block("table", "t", &["v"], header_shaped.clone());
2957 let out = dumps(&doc, false).unwrap();
2958 let parsed = parse(&out)
2959 .unwrap_or_else(|e| panic!("parse failed ({}) for {:?}", e, out));
2960 assert_eq!(
2961 parsed.blocks.len(),
2962 1,
2963 "header-shaped value split the block: {:?}",
2964 out
2965 );
2966 assert_eq!(parsed.blocks[0].rows, header_shaped, "corrupted by {:?}", out);
2967
2968 struct Lcg {
2972 state: u64,
2973 }
2974 impl Lcg {
2975 fn next_int(&mut self, lo: u64, hi: u64) -> u64 {
2976 self.state = (self.state * 1103515245 + 12345) % 2147483648;
2977 lo + (self.state % (hi - lo + 1))
2978 }
2979 }
2980
2981 let alphabet: [&str; 16] = [
2982 "a", "b", " ", "|", "\"", "\\", "\n", "\r", "\t", ".", ":", "#", "0", "1", "true",
2983 "null",
2984 ];
2985 let mut rng = Lcg { state: 20260713 };
2986
2987 for trial in 0..300 {
2988 let num_fields = rng.next_int(1, 4) as usize;
2989 let fields: Vec<String> = (0..num_fields).map(|i| format!("f{}", i)).collect();
2990 let field_refs: Vec<&str> = fields.iter().map(|f| f.as_str()).collect();
2991
2992 let num_rows = rng.next_int(1, 3) as usize;
2993 let mut rows: Vec<Row> = Vec::new();
2994 for _ in 0..num_rows {
2995 let mut row = Row::new();
2996 for f in &fields {
2997 let len = rng.next_int(0, 12) as usize;
2998 let mut s = String::new();
2999 for _ in 0..len {
3000 s.push_str(alphabet[rng.next_int(0, alphabet.len() as u64 - 1) as usize]);
3001 }
3002 row.insert(f.clone(), Value::String(s));
3003 }
3004 rows.push(row);
3005 }
3006
3007 let doc = make_string_block("table", "t", &field_refs, rows.clone());
3008 let out = dumps(&doc, false).unwrap();
3009 let parsed = parse(&out)
3010 .unwrap_or_else(|e| panic!("trial {}: parse failed ({}) for {:?}", trial, e, out));
3011 assert_eq!(
3012 parsed.blocks[0].rows, rows,
3013 "trial {}: {:?} -> {:?}",
3014 trial, rows, out
3015 );
3016 }
3017 }
3018
3019 #[test]
3024 fn test_canonical_blocks_sorted() {
3025 let mut doc = Document::new();
3027
3028 let mut users_block = Block::new("table", "users");
3029 users_block.fields = vec!["id".to_string(), "name".to_string()];
3030 let mut users_row = Row::new();
3031 users_row.insert("id".to_string(), Value::String("2".to_string()));
3032 users_row.insert("name".to_string(), Value::String("Bob".to_string()));
3033 users_block.rows.push(users_row);
3034 doc.blocks.push(users_block);
3035
3036 let mut active_block = Block::new("table", "active_users");
3037 active_block.fields = vec!["id".to_string(), "name".to_string()];
3038 let mut active_row = Row::new();
3039 active_row.insert("id".to_string(), Value::String("1".to_string()));
3040 active_row.insert("name".to_string(), Value::String("Alice".to_string()));
3041 active_block.rows.push(active_row);
3042 doc.blocks.push(active_block);
3043
3044 let mut zulu_block = Block::new("table", "zulu");
3045 zulu_block.fields = vec!["id".to_string(), "name".to_string()];
3046 let mut zulu_row = Row::new();
3047 zulu_row.insert("id".to_string(), Value::String("3".to_string()));
3048 zulu_row.insert("name".to_string(), Value::String("Charlie".to_string()));
3049 zulu_block.rows.push(zulu_row);
3050 doc.blocks.push(zulu_block);
3051
3052 let canonical = dumps_canonical(&doc).unwrap();
3053
3054 assert!(canonical.find("table.active_users").unwrap() < canonical.find("table.users").unwrap());
3056 assert!(canonical.find("table.users").unwrap() < canonical.find("table.zulu").unwrap());
3057 }
3058
3059 #[test]
3060 fn test_canonical_rows_sorted_by_key() {
3061 let mut doc = Document::new();
3063 let mut block = Block::new("table", "items");
3064 block.fields = vec!["id".to_string(), "name".to_string()];
3065
3066 let mut row1 = Row::new();
3067 row1.insert("id".to_string(), Value::String("10".to_string()));
3068 row1.insert("name".to_string(), Value::String("ten".to_string()));
3069 block.rows.push(row1);
3070
3071 let mut row2 = Row::new();
3072 row2.insert("id".to_string(), Value::String("2".to_string()));
3073 row2.insert("name".to_string(), Value::String("two".to_string()));
3074 block.rows.push(row2);
3075
3076 let mut row3 = Row::new();
3077 row3.insert("id".to_string(), Value::String("1".to_string()));
3078 row3.insert("name".to_string(), Value::String("one".to_string()));
3079 block.rows.push(row3);
3080
3081 doc.blocks.push(block);
3082
3083 let canonical = dumps_canonical(&doc).unwrap();
3084 let lines: Vec<&str> = canonical.split('\n').collect();
3085
3086 let data_lines: Vec<&str> = lines.iter()
3088 .filter(|l| !l.contains("table.") && *l != &"id name" && !l.is_empty())
3089 .copied()
3090 .collect();
3091
3092 assert_eq!(data_lines[0], "\"1\" one");
3094 assert_eq!(data_lines[1], "\"10\" ten");
3095 assert_eq!(data_lines[2], "\"2\" two");
3096 }
3097
3098 #[test]
3099 fn test_canonical_null_keys_sort_last() {
3100 let mut doc = Document::new();
3102 let mut block = Block::new("table", "items");
3103 block.fields = vec!["id".to_string(), "name".to_string()];
3104
3105 let mut row1 = Row::new();
3106 row1.insert("id".to_string(), Value::String("2".to_string()));
3107 row1.insert("name".to_string(), Value::String("two".to_string()));
3108 block.rows.push(row1);
3109
3110 let mut row2 = Row::new();
3111 row2.insert("id".to_string(), Value::Null);
3112 row2.insert("name".to_string(), Value::String("orphan".to_string()));
3113 block.rows.push(row2);
3114
3115 let mut row3 = Row::new();
3116 row3.insert("id".to_string(), Value::String("1".to_string()));
3117 row3.insert("name".to_string(), Value::String("one".to_string()));
3118 block.rows.push(row3);
3119
3120 doc.blocks.push(block);
3121
3122 let canonical = dumps_canonical(&doc).unwrap();
3123 let lines: Vec<&str> = canonical.split('\n').collect();
3124
3125 let data_lines: Vec<&str> = lines.iter()
3127 .filter(|l| !l.contains("table.") && *l != &"id name" && !l.is_empty())
3128 .copied()
3129 .collect();
3130
3131 assert_eq!(data_lines[0], "\"1\" one");
3133 assert_eq!(data_lines[1], "\"2\" two");
3134 assert_eq!(data_lines[2], "null orphan");
3135 }
3136
3137 #[test]
3138 fn test_canonical_idempotent() {
3139 let mut doc = Document::new();
3141 let mut block = Block::new("table", "users");
3142 block.fields = vec!["id".to_string(), "name".to_string()];
3143
3144 let mut row1 = Row::new();
3145 row1.insert("id".to_string(), Value::String("2".to_string()));
3146 row1.insert("name".to_string(), Value::String("Bob".to_string()));
3147 block.rows.push(row1);
3148
3149 let mut row2 = Row::new();
3150 row2.insert("id".to_string(), Value::String("1".to_string()));
3151 row2.insert("name".to_string(), Value::String("Alice".to_string()));
3152 block.rows.push(row2);
3153
3154 doc.blocks.push(block);
3155
3156 let canonical1 = dumps_canonical(&doc).unwrap();
3157 let parsed = parse(&canonical1).unwrap();
3158 let canonical2 = dumps_canonical(&parsed).unwrap();
3159
3160 assert_eq!(canonical1, canonical2);
3161 }
3162
3163 #[test]
3164 fn test_canonical_no_alignment() {
3165 let mut doc = Document::new();
3167 let mut block = Block::new("table", "data");
3168 block.fields = vec!["short".to_string(), "very_long_name".to_string()];
3169
3170 let mut row = Row::new();
3171 row.insert("short".to_string(), Value::String("a".to_string()));
3172 row.insert("very_long_name".to_string(), Value::String("b".to_string()));
3173 block.rows.push(row);
3174
3175 doc.blocks.push(block);
3176
3177 let canonical = dumps_canonical(&doc).unwrap();
3178
3179 assert!(canonical.contains("short very_long_name"));
3181 assert!(canonical.contains("a b"));
3182 assert!(!canonical.contains("a "));
3184 }
3185
3186 #[test]
3187 fn test_canonical_with_references() {
3188 let mut doc = Document::new();
3190 let mut block = Block::new("table", "edges");
3191 block.fields = vec!["source".to_string(), "target".to_string()];
3192
3193 let mut row1 = Row::new();
3194 row1.insert("source".to_string(), Value::Reference(Reference::new("2")));
3195 row1.insert("target".to_string(), Value::Reference(Reference::new("b")));
3196 block.rows.push(row1);
3197
3198 let mut row2 = Row::new();
3199 row2.insert("source".to_string(), Value::Reference(Reference::new("1")));
3200 row2.insert("target".to_string(), Value::Reference(Reference::new("a")));
3201 block.rows.push(row2);
3202
3203 doc.blocks.push(block);
3204
3205 let canonical = dumps_canonical(&doc).unwrap();
3206 let lines: Vec<&str> = canonical.split('\n').collect();
3207
3208 let data_lines: Vec<&str> = lines.iter()
3210 .filter(|l| l.starts_with(':'))
3211 .copied()
3212 .collect();
3213
3214 assert_eq!(data_lines[0], ":1 :a");
3216 assert_eq!(data_lines[1], ":2 :b");
3217 }
3218
3219 #[test]
3220 fn test_canonical_golden_fixture() {
3221 let mut doc = Document::new();
3225
3226 let mut edges = Block::new("table", "edges");
3228 edges.fields = vec!["source".to_string(), "target".to_string()];
3229
3230 let mut edge1 = Row::new();
3231 edge1.insert("source".to_string(), Value::Reference(Reference::new("2")));
3232 edge1.insert("target".to_string(), Value::Reference(Reference::new("1")));
3233 edges.rows.push(edge1);
3234
3235 let mut edge2 = Row::new();
3236 edge2.insert("source".to_string(), Value::Reference(Reference::new("1")));
3237 edge2.insert("target".to_string(), Value::Reference(Reference::new("3")));
3238 edges.rows.push(edge2);
3239
3240 doc.blocks.push(edges);
3241
3242 let mut users = Block::new("table", "users");
3244 users.fields = vec!["id".to_string(), "name".to_string(), "active".to_string()];
3245
3246 let mut user1 = Row::new();
3247 user1.insert("id".to_string(), Value::String("2".to_string()));
3248 user1.insert("name".to_string(), Value::String("Bob".to_string()));
3249 user1.insert("active".to_string(), Value::Bool(true));
3250 users.rows.push(user1);
3251
3252 let mut user2 = Row::new();
3253 user2.insert("id".to_string(), Value::String("1".to_string()));
3254 user2.insert("name".to_string(), Value::String("Alice".to_string()));
3255 user2.insert("active".to_string(), Value::Bool(true));
3256 users.rows.push(user2);
3257
3258 let mut user3 = Row::new();
3259 user3.insert("id".to_string(), Value::String("3".to_string()));
3260 user3.insert("name".to_string(), Value::String("Charlie".to_string()));
3261 user3.insert("active".to_string(), Value::Bool(false));
3262 users.rows.push(user3);
3263
3264 doc.blocks.push(users);
3265
3266 let canonical = dumps_canonical(&doc).unwrap();
3267
3268 let expected_lines = vec![
3271 "table.edges",
3272 "source target",
3273 ":1 :3",
3274 ":2 :1",
3275 "",
3276 "table.users",
3277 "id active name",
3278 "\"1\" true Alice",
3279 "\"2\" true Bob",
3280 "\"3\" false Charlie",
3281 ];
3282 let expected = expected_lines.join("\n");
3283
3284 assert_eq!(canonical, expected, "\nExpected:\n{}\n\nGot:\n{}", expected, canonical);
3285 }
3286
3287 #[test]
3288 fn test_canonical_isonl_blocks_sorted() {
3289 let mut doc = Document::new();
3291
3292 let mut zebras = Block::new("table", "zebras");
3293 zebras.fields = vec!["id".to_string()];
3294 let mut row = Row::new();
3295 row.insert("id".to_string(), Value::String("1".to_string()));
3296 zebras.rows.push(row);
3297 doc.blocks.push(zebras);
3298
3299 let mut aardvarks = Block::new("table", "aardvarks");
3300 aardvarks.fields = vec!["id".to_string()];
3301 let mut row = Row::new();
3302 row.insert("id".to_string(), Value::String("2".to_string()));
3303 aardvarks.rows.push(row);
3304 doc.blocks.push(aardvarks);
3305
3306 let canonical_isonl = dumps_canonical_isonl(&doc).unwrap();
3307 let lines: Vec<&str> = canonical_isonl.split('\n').collect();
3308
3309 assert!(lines[0].contains("table.aardvarks"));
3311 assert!(lines[1].contains("table.zebras"));
3312 }
3313
3314 #[test]
3315 fn test_canonical_isonl_rows_sorted() {
3316 let mut doc = Document::new();
3318 let mut block = Block::new("table", "items");
3319 block.fields = vec!["id".to_string(), "val".to_string()];
3320
3321 let mut row1 = Row::new();
3322 row1.insert("id".to_string(), Value::String("c".to_string()));
3323 row1.insert("val".to_string(), Value::String("three".to_string()));
3324 block.rows.push(row1);
3325
3326 let mut row2 = Row::new();
3327 row2.insert("id".to_string(), Value::String("a".to_string()));
3328 row2.insert("val".to_string(), Value::String("one".to_string()));
3329 block.rows.push(row2);
3330
3331 let mut row3 = Row::new();
3332 row3.insert("id".to_string(), Value::String("b".to_string()));
3333 row3.insert("val".to_string(), Value::String("two".to_string()));
3334 block.rows.push(row3);
3335
3336 doc.blocks.push(block);
3337
3338 let canonical_isonl = dumps_canonical_isonl(&doc).unwrap();
3339 let lines: Vec<&str> = canonical_isonl.split('\n').collect();
3340
3341 let idx_a = lines.iter().position(|l| l.contains("a one")).unwrap();
3344 let idx_b = lines.iter().position(|l| l.contains("b two")).unwrap();
3345 let idx_c = lines.iter().position(|l| l.contains("c three")).unwrap();
3346 assert!(idx_a < idx_b && idx_b < idx_c);
3347 }
3348
3349 #[test]
3350 fn test_canonical_empty_string_handling() {
3351 let mut doc = Document::new();
3353 let mut block = Block::new("table", "test");
3354 block.fields = vec!["id".to_string(), "name".to_string()];
3355
3356 let mut row1 = Row::new();
3357 row1.insert("id".to_string(), Value::String("1".to_string()));
3358 row1.insert("name".to_string(), Value::String("".to_string()));
3359 block.rows.push(row1);
3360
3361 let mut row2 = Row::new();
3362 row2.insert("id".to_string(), Value::String("2".to_string()));
3363 row2.insert("name".to_string(), Value::String("value".to_string()));
3364 block.rows.push(row2);
3365
3366 doc.blocks.push(block);
3367
3368 let canonical = dumps_canonical(&doc).unwrap();
3369
3370 assert!(canonical.contains("\"\""));
3372 }
3373
3374 #[test]
3375 fn test_canonical_field_info_preserved() {
3376 let mut doc = Document::new();
3378 let mut block = Block::new("table", "typed");
3379 block.fields = vec!["id".to_string(), "count".to_string()];
3380 block.field_info = vec![
3381 FieldInfo::with_type("id", "string"),
3382 FieldInfo::with_type("count", "int"),
3383 ];
3384
3385 let mut row = Row::new();
3386 row.insert("id".to_string(), Value::String("1".to_string()));
3387 row.insert("count".to_string(), Value::Int(42));
3388 block.rows.push(row);
3389
3390 doc.blocks.push(block);
3391
3392 let canonical = dumps_canonical(&doc).unwrap();
3393
3394 assert!(canonical.contains("id:string count:int"));
3395 }
3396
3397 #[test]
3398 fn test_canonical_field_sort_golden_fixture() {
3399 let mut doc = Document::new();
3403
3404 let mut no_id = Block::new("table", "no_id");
3406 no_id.fields = vec!["name".to_string(), "city".to_string(), "age".to_string()]; let mut row = Row::new();
3408 row.insert("name".to_string(), Value::String("Charlie".to_string()));
3409 row.insert("city".to_string(), Value::String("New York".to_string()));
3410 row.insert("age".to_string(), Value::Int(30));
3411 no_id.rows.push(row);
3412 doc.blocks.push(no_id);
3413
3414 let mut scrambled = Block::new("table", "scrambled");
3416 scrambled.fields = vec![
3417 "score".to_string(),
3418 "active".to_string(),
3419 "id".to_string(),
3420 "email".to_string(),
3421 "name".to_string(),
3422 ]; let mut row1 = Row::new();
3425 row1.insert("score".to_string(), Value::Float(95.5));
3426 row1.insert("active".to_string(), Value::Bool(true));
3427 row1.insert("id".to_string(), Value::Int(1));
3428 row1.insert("email".to_string(), Value::String("alice@example.com".to_string()));
3429 row1.insert("name".to_string(), Value::String("Alice".to_string()));
3430 scrambled.rows.push(row1);
3431
3432 let mut row2 = Row::new();
3433 row2.insert("score".to_string(), Value::Float(87.3));
3434 row2.insert("active".to_string(), Value::Bool(false));
3435 row2.insert("id".to_string(), Value::Int(2));
3436 row2.insert("email".to_string(), Value::String("bob@example.com".to_string()));
3437 row2.insert("name".to_string(), Value::String("Bob".to_string()));
3438 scrambled.rows.push(row2);
3439
3440 doc.blocks.push(scrambled);
3441
3442 let mut utf16_div = Block::new("table", "utf16_divergence");
3444 utf16_div.fields = vec![
3445 "😀field".to_string(),
3446 "id".to_string(),
3447 "Afield".to_string(),
3448 ]; let mut row3 = Row::new();
3451 row3.insert("😀field".to_string(), Value::String("non-BMP emoji (U+1F600 starts 0xF0 in UTF-8)".to_string()));
3452 row3.insert("id".to_string(), Value::Int(101));
3453 row3.insert("Afield".to_string(), Value::String("fullwidth A (U+FF21 is 0xEF in UTF-8)".to_string()));
3454 utf16_div.rows.push(row3);
3455
3456 doc.blocks.push(utf16_div);
3457
3458 let canonical = dumps_canonical(&doc).unwrap();
3459
3460 assert!(
3462 canonical.contains("table.no_id\nage city name"),
3463 "no_id: fields should be sorted as 'age city name'"
3464 );
3465
3466 assert!(
3467 canonical.contains("table.scrambled\nid active email name score"),
3468 "scrambled: fields should be sorted as 'id active email name score'"
3469 );
3470
3471 assert!(
3472 canonical.contains("table.utf16_divergence\nid Afield 😀field"),
3473 "utf16_divergence: CRITICAL - Afield (0xEF) should come before 😀field (0xF0)"
3474 );
3475 }
3476
3477 #[test]
3478 #[cfg(feature = "serde")]
3479 fn test_canonical_field_sort_json_golden_fixture() {
3480 let json_input = r#"{
3482 "scrambled": [
3483 {
3484 "score": 95.5,
3485 "active": true,
3486 "id": 1,
3487 "email": "alice@example.com",
3488 "name": "Alice"
3489 },
3490 {
3491 "name": "Bob",
3492 "email": "bob@example.com",
3493 "score": 87.3,
3494 "id": 2,
3495 "active": false
3496 }
3497 ],
3498 "no_id": [
3499 {
3500 "city": "New York",
3501 "age": 30,
3502 "name": "Charlie"
3503 }
3504 ],
3505 "utf16_divergence": [
3506 {
3507 "id": 101,
3508 "😀field": "non-BMP emoji (U+1F600 starts 0xF0 in UTF-8)",
3509 "Afield": "fullwidth A (U+FF21 is 0xEF in UTF-8)"
3510 }
3511 ],
3512 "users_order_1": [
3513 {
3514 "id": 1001,
3515 "name": "David",
3516 "email": "david@example.com"
3517 }
3518 ],
3519 "users_order_2": [
3520 {
3521 "email": "eve@example.com",
3522 "name": "Eve",
3523 "id": 1002
3524 }
3525 ],
3526 "empty": [],
3527 "single_row": [
3528 {
3529 "id": 9999,
3530 "value": "only_one"
3531 }
3532 ]
3533}"#;
3534
3535 let canonical_result = json_to_ison_canonical(json_input);
3537 assert!(canonical_result.is_ok(), "JSON to canonical ISON conversion should succeed");
3538
3539 let canonical = canonical_result.unwrap();
3540
3541 assert!(canonical.contains("table.no_id\nage city name"),
3545 "no_id: fields should be 'age city name' (alphabetically sorted by UTF-8 bytes)");
3546
3547 assert!(canonical.contains("table.scrambled\nid active email name score"),
3549 "scrambled: fields should be 'id active email name score' (id first, then sorted)");
3550
3551 assert!(canonical.contains("table.utf16_divergence\nid Afield 😀field"),
3556 "UTF-16 DIVERGENCE: Afield (0xEF...) must come before 😀field (0xF0...) by UTF-8 byte order");
3557
3558 assert!(canonical.contains("table.single_row\nid value"),
3560 "single_row: fields should be 'id value' (id first, then value)");
3561
3562 assert!(canonical.contains("table.users_order_1\nid email name"),
3564 "users_order_1: fields should be 'id email name'");
3565 assert!(canonical.contains("table.users_order_2\nid email name"),
3566 "users_order_2: fields should also be 'id email name' (field order is canonical, not input order)");
3567 }
3568}