mech_core/
nodes.rs

1use crate::*; 
2#[cfg(feature = "no_std")]
3use core::cmp::Ordering;
4#[cfg(not(feature = "no_std"))] 
5use std::cmp::Ordering;
6
7#[cfg(feature = "serde")]
8pub fn compress_and_encode<T: serde::Serialize>(tree: &T) -> Result<String, Box<dyn std::error::Error>> {
9  let serialized_code = bincode::serde::encode_to_vec(tree,bincode::config::standard())?;
10  let mut compressed = Vec::new();
11  brotli::CompressorWriter::new(&mut compressed, 9, 4096, 22)
12      .write(&serialized_code)?;
13  Ok(base64::encode(compressed))
14}
15
16#[cfg(feature = "serde")]
17pub fn decode_and_decompress<T: serde::de::DeserializeOwned>(encoded: &str) -> Result<T, Box<dyn std::error::Error>> {
18  let decoded = base64::decode(encoded)?;
19  
20  let mut decompressed = Vec::new();
21  brotli::Decompressor::new(Cursor::new(decoded), 4096)
22      .read_to_end(&mut decompressed)?;
23  
24  let (decoded,red) = bincode::serde::decode_from_slice(&decompressed,bincode::config::standard())?;
25
26  Ok(decoded)
27}
28#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
29#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord)]
30pub struct SourceLocation {
31  pub row: usize,
32  pub col: usize,
33}
34
35impl PartialOrd for SourceLocation {
36  fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
37    if self.row < other.row {
38      Some(Ordering::Less)
39    } else if self.row > other.row {
40      Some(Ordering::Greater)
41    } else {
42      self.col.partial_cmp(&other.col)
43    }
44  }
45}
46
47impl fmt::Debug for SourceLocation {
48  #[inline]
49  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
50    write!(f, "{}:{}", self.row, self.col);
51    Ok(())
52  }
53}
54
55#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
56#[derive(Clone, Hash, PartialEq, Eq)]
57pub struct SourceRange {
58  pub start: SourceLocation,
59  pub end:   SourceLocation,
60}
61
62/// Coordinates in SourceRange are 1-indexed, i.e. they directly translate
63/// human's view to line and column numbers.  Having value 0 means the 
64/// range is not initialized.
65impl Default for SourceRange {
66  fn default() -> Self {
67    SourceRange {
68      start: SourceLocation { row: 0, col: 0 },
69      end:   SourceLocation { row: 0, col: 0 },
70    }
71  }
72}
73
74impl fmt::Debug for SourceRange {
75  #[inline]
76  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
77    write!(f, "[{:?}, {:?})", self.start, self.end);
78    Ok(())
79  }
80}
81
82pub fn merge_src_range(r1: SourceRange, r2: SourceRange) -> SourceRange {
83  SourceRange {
84    start: r1.start.min(r2.start),
85    end:   r2.end.max(r2.end),
86  }
87}
88
89pub trait Recoverable: Sized {
90  fn error_placeholder(skipped_tokens: Token, range: SourceRange) -> Self;
91}
92
93#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
94#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
95pub enum TokenKind {
96  AbstractSigil, Alpha, Ampersand, Any, Apostrophe, AssignOperator, Asterisk, AsyncTransitionOperator, At,
97  Backslash, Bar, BoxDrawing,
98  Caret, CarriageReturn, CarriageReturnNewLine, Colon, CodeBlock, Comma,
99  Dash, DefineOperator, Digit, Dollar,
100  Emoji, EmphasisSigil, Empty, Equal, EquationSigil, Error, ErrorSigil, Exclamation, 
101  False, FloatLeft, FloatRight, FootnotePrefix,
102  Grave, GraveCodeBlockSigil,
103  HashTag, HighlightSigil, HttpPrefix,
104  IdeaSigil, Identifier, ImgPrefix, InfoSigil, InlineCode, 
105  LeftAngle, LeftBrace, LeftBracket, LeftParenthesis,
106  Newline, Not, Number,
107  OutputOperator,
108  Percent, Period, Plus,
109  QueryOperator, Question, QuestionSigil, Quote, QuoteSigil,
110  RightAngle, RightBrace, RightBracket, RightParenthesis,
111  SectionSigil, Semicolon, Space, Slash, String, StrikeSigil, StrongSigil, SuccessSigil,
112  Tab, Text, Tilde, TildeCodeBlockSigil, Title, TransitionOperator, True,
113  UnderlineSigil, Underscore,
114  WarningSigil,
115}
116
117#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
118#[derive(Clone, Hash, PartialEq, Eq)]
119pub struct Token { 
120  pub kind: TokenKind, 
121  pub chars: Vec<char>, 
122  pub src_range: SourceRange 
123}
124
125impl fmt::Debug for Token {
126  #[inline]
127  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
128    write!(f, "{:?}:{:?}:{:?}", self.kind, String::from_iter(self.chars.iter().cloned()), self.src_range);
129    Ok(())
130  }
131}
132
133impl Default for Token {
134  fn default() -> Self {
135    Token{
136      kind: TokenKind::Empty,
137      chars: vec![],
138      src_range: SourceRange::default(),
139    }
140  }
141}
142
143impl Token {
144
145  pub fn new(kind: TokenKind, src_range: SourceRange, chars: Vec<char>) -> Token {
146    Token{kind, chars, src_range}
147  }
148
149  pub fn to_string(&self) -> String {
150    self.chars.iter().collect()
151  }
152
153  pub fn merge_tokens(tokens: &mut Vec<Token>) -> Option<Token> {
154    if tokens.len() == 0 {
155      None
156    } else if tokens.len() == 1 {
157      Some(tokens[0].clone())
158    } else {
159      let first = tokens[0].src_range.clone();
160      let kind = tokens[0].kind.clone();
161      let last = tokens.last().unwrap().src_range.clone();
162      let src_range = merge_src_range(first, last);
163      let chars: Vec<char> = tokens.iter_mut().fold(vec![],|mut m, ref mut t| {m.append(&mut t.chars.clone()); m});
164      let merged_token = Token{kind, chars, src_range};
165      Some(merged_token)
166    }
167  }
168}
169
170#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
171#[derive(Clone, Debug, Hash, PartialEq, Eq)]
172pub struct TableOfContents {
173    pub title: Option<Title>,
174    pub sections: Vec<Section>,
175}
176
177#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
178#[derive(Clone, Debug, Hash, PartialEq, Eq)]
179pub struct Program {
180  pub title: Option<Title>,
181  pub body: Body,
182}
183
184impl Program {
185  pub fn tokens(&self) -> Vec<Token> {
186    /*let mut title_tokens = match self.title.tokens() {
187      Some(tkns) => tkns,
188      None => vec![],
189    };*/
190    let body_tokens = self.body.tokens();
191    body_tokens
192  }
193
194  pub fn table_of_contents(&self) -> TableOfContents {
195    let mut sections = vec![];
196    for s in &self.body.sections {
197      sections.push(s.table_of_contents());
198    }
199    TableOfContents {
200      title: self.title.clone(),
201      sections,
202    }
203  }
204
205}
206
207#[cfg(feature = "pretty_print")]
208impl PrettyPrint for Program {
209fn pretty_print(&self) -> String {
210    #[cfg(feature = "serde")]
211    let json_string = serde_json::to_string_pretty(self).unwrap();
212    #[cfg(not(feature = "serde"))]
213    let json_string = "Error: Enable Serde to prettt print trees.".to_string();
214
215    let depth = |line: &str|->usize{line.chars().take_while(|&c| c == ' ').count()};
216    let mut result = String::new();
217    let lines: Vec<&str> = json_string.lines().collect();
218    result.push_str("Program\n");
219    for (index, line) in lines.iter().enumerate() {
220      let trm = line.trim();
221      if trm == "}" || 
222         trm == "},"|| 
223         trm == "{" || 
224         trm == "[" || 
225         trm == "],"|| 
226         trm == "]" {
227        continue;
228      }
229  
230      // Count leading spaces to determine depth
231      let d = depth(line);
232      // Construct the tree-like prefix
233      let mut prefix = String::new();
234      for _ in 0..d {
235        prefix.push_str(" ");
236      }
237      if index == lines.len() {
238        prefix.push_str("└ ");
239      } else {
240        if depth(lines[index + 1]) != d {
241          prefix.push_str("└ ");
242        } else {
243          prefix.push_str("├ ");
244        }
245      }
246      let trm = line.trim();
247      let trm = trm.trim_end_matches('{')
248                    .trim_start_matches('"')
249                    .trim_end_matches(':')
250                    .trim_end_matches('"')
251                    .trim_end_matches('[');
252      prefix.push_str(trm);
253  
254      // Append formatted line to result
255      result.push_str(&prefix);
256      result.push('\n');
257      result = result.replace("\":", "");
258    }
259    let mut indexed_str = IndexedString::new(&result);
260    'rows: for i in 0..indexed_str.rows {
261      let rowz = &indexed_str.index_map[i];
262      for j in 0..rowz.len() {
263        let c = match indexed_str.get(i,j) {
264          Some(c) => c,
265          None => continue,
266        };
267        if c == '└' {
268          for k in i+1..indexed_str.rows {
269            match indexed_str.get(k,j) {
270              Some(c2) => {
271                if c2 == '└' {
272                  indexed_str.set(i,j,'├');
273                  for l in i+1..k {
274                    match indexed_str.get(l,j) {
275                      Some(' ') => {indexed_str.set(l,j,'│');},
276                      Some('└') => {indexed_str.set(l,j,'├');},
277                      _ => (),
278                    }
279                  }
280                } else if c2 == ' ' {
281                  continue;
282                } else {
283                  continue 'rows;
284                }
285              },
286              None => continue,
287            }
288  
289          }
290        } else if c == ' ' || c == '│' {
291          continue;
292        } else {
293          continue 'rows;
294        }
295      }
296    }
297    indexed_str.to_string()
298  }
299}
300
301#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
302#[derive(Clone, Debug, Hash, PartialEq, Eq)]
303pub struct Title {
304  pub text: Token,
305}
306
307impl Title {
308
309  pub fn to_string(&self) -> String {
310    self.text.to_string()
311  }
312
313}
314
315#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
316#[derive(Clone, Debug, Hash, PartialEq, Eq)]
317pub struct Body {
318  pub sections: Vec<Section>,
319}
320
321impl Body {
322  pub fn tokens(&self) -> Vec<Token> {
323    let mut out = vec![];
324    for s in &self.sections {
325      let mut tkns = s.tokens();
326      out.append(&mut tkns);
327    }
328    out
329  }
330}
331
332#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
333#[derive(Clone, Debug, Hash, PartialEq, Eq)]
334pub enum ColumnAlignment {
335  Left,
336  Center,
337  Right,
338}
339
340#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
341#[derive(Clone, Debug, Hash, PartialEq, Eq)]
342pub struct MarkdownTable {
343  pub header: Vec<Paragraph>,
344  pub rows: Vec<Vec<Paragraph>>,
345  pub alignment: Vec<ColumnAlignment>,
346}
347
348#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
349#[derive(Clone, Debug, Hash, PartialEq, Eq)]
350pub struct Subtitle {
351  pub text: Paragraph,
352  pub level: u8,
353}
354
355impl Subtitle {
356  pub fn to_string(&self) -> String {
357    self.text.to_string()
358  }
359}
360
361#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
362#[derive(Clone, Debug, Hash, PartialEq, Eq)]
363pub struct Section {
364  pub subtitle: Option<Subtitle>,
365  pub elements: Vec<SectionElement>,
366}
367
368impl Section {
369
370  pub fn table_of_contents(&self) -> Section {
371    let elements: Vec<_> = self.elements.iter()
372      .filter(|e| matches!(e, SectionElement::Subtitle(_)))
373      .cloned()
374      .collect();
375    Section {
376      subtitle: self.subtitle.clone(),
377      elements,
378    }
379  }
380
381  pub fn tokens(&self) -> Vec<Token> {
382    let mut out = vec![];
383    for s in &self.elements {
384      let mut tkns = s.tokens();
385      out.append(&mut tkns);
386    }
387    out
388  }
389}
390
391#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
392#[derive(Clone, Debug, Hash, PartialEq, Eq)]
393pub struct Grammar {
394  pub rules: Vec<Rule>,
395}
396
397#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
398#[derive(Clone, Debug, Hash, PartialEq, Eq)]
399pub struct GrammarIdentifier {
400  pub name: Token,
401}
402
403impl GrammarIdentifier {
404  pub fn tokens(&self) -> Vec<Token> {
405    vec![self.name.clone()]
406  }
407
408  pub fn to_string(&self) -> String {
409    self.name.to_string()
410  }
411}
412
413#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
414#[derive(Clone, Debug, Hash, PartialEq, Eq)]
415pub struct Rule {
416  pub name: GrammarIdentifier,
417  pub expr: GrammarExpression,
418}
419
420#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
421#[derive(Clone, Debug, Hash, PartialEq, Eq)]
422pub enum GrammarExpression {
423  Choice(Vec<GrammarExpression>),
424  Definition(GrammarIdentifier),
425  Group(Box<GrammarExpression>),
426  List(Box<GrammarExpression>, Box<GrammarExpression>),
427  Not(Box<GrammarExpression>),
428  Optional(Box<GrammarExpression>),
429  Peek(Box<GrammarExpression>),
430  Repeat0(Box<GrammarExpression>),
431  Repeat1(Box<GrammarExpression>),
432  Range(Token,Token),
433  Sequence(Vec<GrammarExpression>),
434  Terminal(Token),
435}
436
437impl GrammarExpression {
438  pub fn tokens(&self) -> Vec<Token> {
439    match self {
440      GrammarExpression::Choice(exprs) => {
441        let mut tokens = vec![];
442        for expr in exprs {
443          tokens.append(&mut expr.tokens());
444        }
445        tokens
446      }
447      GrammarExpression::Definition(id) => id.tokens(),
448      GrammarExpression::Group(expr) => expr.tokens(),
449      GrammarExpression::List(expr1, expr2) => {
450        let mut tokens = expr1.tokens();
451        tokens.append(&mut expr2.tokens());
452        tokens
453      }
454      GrammarExpression::Not(expr) => expr.tokens(),
455      GrammarExpression::Optional(expr) => expr.tokens(),
456      GrammarExpression::Peek(expr) => expr.tokens(),
457      GrammarExpression::Repeat0(expr) => expr.tokens(),
458      GrammarExpression::Repeat1(expr) => expr.tokens(),
459      GrammarExpression::Range(t1, t2) => vec![t1.clone(), t2.clone()],
460      GrammarExpression::Sequence(exprs) => {
461        let mut tokens = vec![];
462        for expr in exprs {
463          tokens.append(&mut expr.tokens());
464        }
465        tokens
466      }
467      GrammarExpression::Terminal(t) => vec![t.clone()],
468    }
469  }
470
471
472}
473
474#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
475#[derive(Clone, Debug, Hash, PartialEq, Eq)]
476pub struct Citation {
477  pub id: Token,
478  pub text: Paragraph,
479}
480
481impl Citation {
482  pub fn to_string(&self) -> String {
483    format!("[{}]: {}", self.id.to_string(), self.text.to_string())
484  }
485}
486
487// Stores code block configuration
488#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
489#[derive(Clone, Debug, Hash, PartialEq, Eq)]
490pub struct BlockConfig {
491  pub namespace_str: String,
492  pub namespace: u64,
493  pub disabled: bool,
494  pub hidden: bool,
495}
496
497pub type Footnote = (Token, Vec<Paragraph>);
498
499#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
500#[derive(Clone, Debug, Hash, PartialEq, Eq)]
501pub enum FloatDirection {
502  Left,
503  Right,
504}
505
506#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
507#[derive(Clone, Debug, Hash, PartialEq, Eq)]
508pub struct FencedMechCode {
509  pub code: Vec<(MechCode,Option<Comment>)>,
510  pub config: BlockConfig,
511  pub options: Option<OptionMap>,
512}
513
514#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
515#[derive(Clone, Debug, Hash, PartialEq, Eq)]
516pub enum SectionElement {
517  Abstract(Vec<Paragraph>),
518  QuoteBlock(Vec<Paragraph>),
519  InfoBlock(Vec<Paragraph>),
520  SuccessBlock(Vec<Paragraph>),
521  IdeaBlock(Vec<Paragraph>),
522  WarningBlock(Vec<Paragraph>),
523  ErrorBlock(Vec<Paragraph>),
524  QuestionBlock(Vec<Paragraph>),
525  Citation(Citation),
526  CodeBlock(Token),
527  Comment(Comment),
528  Diagram(Token),
529  Equation(Token),
530  FencedMechCode(FencedMechCode),
531  Float((Box<SectionElement>, FloatDirection)),
532  Footnote(Footnote),
533  Grammar(Grammar),
534  Image(Image),
535  List(MDList),
536  MechCode(Vec<(MechCode,Option<Comment>)>),
537  Paragraph(Paragraph),
538  Subtitle(Subtitle),
539  Table(MarkdownTable),
540  ThematicBreak,
541  Error(Token, SourceRange),
542}
543
544impl Recoverable for SectionElement {
545  fn error_placeholder(skipped_tokens: Token, range: SourceRange) -> Self {
546    SectionElement::Error(skipped_tokens, range)
547  }
548}
549
550impl SectionElement {
551
552  pub fn tokens(&self) -> Vec<Token> {
553    match self {
554      SectionElement::FencedMechCode(c) => {
555        let mut tokens = vec![];
556        for (c, _) in &c.code {
557          tokens.append(&mut c.tokens());
558        }
559        tokens
560      }
561      SectionElement::MechCode(codes) => {
562        let mut tokens = vec![];
563        for (code, _) in codes {
564          tokens.append(&mut code.tokens());
565        }
566        tokens
567      }
568      SectionElement::Abstract(paragraphs)
569      | SectionElement::QuoteBlock(paragraphs)
570      | SectionElement::InfoBlock(paragraphs)
571      | SectionElement::SuccessBlock(paragraphs)
572      | SectionElement::IdeaBlock(paragraphs)
573      | SectionElement::WarningBlock(paragraphs)
574      | SectionElement::ErrorBlock(paragraphs)
575      | SectionElement::QuestionBlock(paragraphs) => {
576        let mut tokens = vec![];
577        for paragraph in paragraphs {
578          tokens.append(&mut paragraph.tokens());
579        }
580        tokens
581      }
582      SectionElement::Citation(citation) => citation.text.tokens(),
583      SectionElement::CodeBlock(token)
584      | SectionElement::Diagram(token)
585      | SectionElement::Equation(token) => vec![token.clone()],
586      SectionElement::Comment(comment) => comment.tokens(),
587      SectionElement::Float((element, _)) => element.tokens(),
588      SectionElement::Footnote((_, paragraphs)) => {
589        let mut tokens = vec![];
590        for paragraph in paragraphs {
591          tokens.append(&mut paragraph.tokens());
592        }
593        tokens
594      }
595        SectionElement::Grammar(grammar) => {
596        let mut tokens = vec![];
597        for rule in &grammar.rules {
598          tokens.append(&mut rule.name.tokens());
599          tokens.append(&mut rule.expr.tokens());
600        }
601        tokens
602      }
603      SectionElement::Image(image) => {
604        let mut tokens = vec![image.src.clone()];
605        if let Some(caption) = &image.caption {
606          tokens.append(&mut caption.tokens());
607        }
608        tokens
609      }
610      SectionElement::List(list) => match list {
611      MDList::Unordered(items) => {
612        let mut tokens = vec![];
613        for ((_, paragraph), sublist) in items {
614        tokens.append(&mut paragraph.tokens());
615        if let Some(sublist) = sublist {
616          tokens.append(&mut sublist.tokens());
617        }
618        }
619        tokens
620      }
621      MDList::Ordered(ordered_list) => {
622        let mut tokens = ordered_list.start.tokens();
623        for ((_, paragraph), sublist) in &ordered_list.items {
624        tokens.append(&mut paragraph.tokens());
625        if let Some(sublist) = sublist {
626          tokens.append(&mut sublist.tokens());
627        }
628        }
629        tokens
630      }
631      MDList::Check(items) => {
632        let mut tokens = vec![];
633        for ((_, paragraph), sublist) in items {
634        tokens.append(&mut paragraph.tokens());
635        if let Some(sublist) = sublist {
636          tokens.append(&mut sublist.tokens());
637        }
638        }
639        tokens
640      }
641      },
642      SectionElement::Paragraph(paragraph) => paragraph.tokens(),
643      SectionElement::Subtitle(subtitle) => subtitle.text.tokens(),
644      SectionElement::Table(table) => {
645        let mut tokens = vec![];
646        for header in &table.header {
647          tokens.append(&mut header.tokens());
648        }
649        for row in &table.rows {
650          for column in row {
651          tokens.append(&mut column.tokens());
652          }
653        }
654        tokens
655      }
656      SectionElement::ThematicBreak => vec![],
657      SectionElement::Error(token, _) => vec![token.clone()],
658    }
659  }
660}
661
662
663#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
664#[derive(Clone, Debug, Hash, PartialEq, Eq)]
665pub struct OptionMap {
666  pub elements: Vec<(Identifier, MechString)>,
667}
668
669
670#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
671#[derive(Clone, Debug, Hash, PartialEq, Eq)]
672pub struct Image {
673  pub src: Token,
674  pub caption: Option<Paragraph>,
675  pub style: Option<OptionMap>,
676}
677
678impl Image {
679  pub fn to_string(&self) -> String {
680    let caption = match &self.caption {
681      Some(c) => c.to_string(),
682      None => "".to_string(),
683    };
684    format!("![{}]({})", caption, self.src.to_string())
685  }
686}
687
688pub type UnorderedList = Vec<((Option<Token>,Paragraph),Option<MDList>)>;
689pub type CheckList = Vec<((bool,Paragraph),Option<MDList>)>;
690
691#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
692#[derive(Clone, Debug, Hash, PartialEq, Eq)]
693pub struct OrderedList {
694  pub start: Number,
695  pub items: Vec<((Number,Paragraph),Option<MDList>)>,
696}
697
698#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
699#[derive(Clone, Debug, Hash, PartialEq, Eq)]
700pub enum MDList {
701  Unordered(UnorderedList),
702  Ordered(OrderedList),
703  Check(CheckList)
704}
705
706impl MDList {
707
708  pub fn tokens(&self) -> Vec<Token> {
709    match self {
710      MDList::Unordered(items) => {
711        let mut tokens = vec![];
712        for ((_, paragraph), sublist) in items {
713          tokens.append(&mut paragraph.tokens());
714          if let Some(sublist) = sublist {
715            tokens.append(&mut sublist.tokens());
716          }
717        }
718        tokens
719      }
720      MDList::Ordered(ordered_list) => {
721        let mut tokens = ordered_list.start.tokens();
722        for ((_, paragraph), sublist) in &ordered_list.items {
723          tokens.append(&mut paragraph.tokens());
724          if let Some(sublist) = sublist {
725            tokens.append(&mut sublist.tokens());
726          }
727        }
728        tokens
729      }
730      MDList::Check(items) => {
731        let mut tokens = vec![];
732        for ((_, paragraph), sublist) in items {
733          tokens.append(&mut paragraph.tokens());
734          if let Some(sublist) = sublist {
735            tokens.append(&mut sublist.tokens());
736          }
737        }
738        tokens
739      }
740    }
741  }
742
743}
744
745#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
746#[derive(Clone, Debug, Hash, PartialEq, Eq)]
747pub enum MechCode {
748  Comment(Comment),
749  Expression(Expression),
750  //FsmImplementation(FsmImplementation),
751  //FsmSpecification(FsmSpecification),
752  FunctionDefine(FunctionDefine),
753  Statement(Statement),
754  Error(Token, SourceRange),
755}
756
757impl Recoverable for MechCode {
758  fn error_placeholder(skipped_tokens: Token, range: SourceRange) -> Self {
759    MechCode::Error(skipped_tokens, range)
760  }
761}
762
763impl MechCode {
764  pub fn tokens(&self) -> Vec<Token> {
765    match self {
766      MechCode::Expression(x) => x.tokens(),
767      MechCode::Statement(x) => x.tokens(),
768      MechCode::Comment(x) => x.tokens(),
769      MechCode::Error(t,_) => vec![t.clone()],
770      _ => todo!(),
771      //FunctionDefine(x) => x.tokens(),
772      //FsmSpecification(x) => x.tokens(),
773      //FsmImplementation(x) => x.tokens(),
774    }
775  }
776}
777
778#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
779#[derive(Clone, Debug, Hash, PartialEq, Eq)]
780pub struct FunctionDefine {
781  pub name: Identifier,
782  pub input: Vec<FunctionArgument>,
783  pub output: Vec<FunctionArgument>,
784  pub statements: Vec<Statement>,
785}
786
787#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
788#[derive(Clone, Debug, Hash, PartialEq, Eq)]
789pub struct FunctionArgument {
790  pub name: Identifier,
791  pub kind: KindAnnotation,
792}
793
794impl FunctionArgument {
795  pub fn tokens(&self) -> Vec<Token> {
796    let mut tokens = self.name.tokens();
797    tokens.append(&mut self.kind.tokens());
798    tokens
799  }
800}
801
802#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
803#[derive(Clone, Debug, Hash, PartialEq, Eq)]
804pub struct FsmImplementation {
805  pub name: Identifier,
806  pub input: Vec<Identifier>,
807  pub start: Pattern,
808  pub arms: Vec<FsmArm>,
809}
810
811#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
812#[derive(Clone, Debug, Hash, PartialEq, Eq)]
813pub enum FsmArm {
814  Guard(Pattern,Vec<Guard>),
815  Transition(Pattern,Vec<Transition>),
816}
817
818#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
819#[derive(Clone, Debug, Hash, PartialEq, Eq)]
820pub struct Guard { 
821  pub condition: Pattern,
822  pub transitions: Vec<Transition>,
823}
824
825#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
826#[derive(Clone, Debug, Hash, PartialEq, Eq)]
827pub enum Transition {
828  Async(Pattern),
829  CodeBlock(Vec<(MechCode, Option<Comment>)>),
830  Next(Pattern),
831  Output(Pattern),
832  Statement(Statement),
833}
834
835#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
836#[derive(Clone, Debug, Hash, PartialEq, Eq)]
837pub enum Pattern {
838  Expression(Expression),
839  Formula(Factor),
840  TupleStruct(PatternTupleStruct),
841  Wildcard,
842}
843
844#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
845#[derive(Clone, Debug, Hash, PartialEq, Eq)]
846pub struct PatternTupleStruct {
847  pub name: Identifier,
848  pub patterns: Vec<Pattern>,
849}
850
851pub type PatternTuple = Vec<Pattern>;
852
853#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
854#[derive(Clone, Debug, Hash, PartialEq, Eq)]
855pub struct FsmSpecification {
856  pub name: Identifier,
857  pub input: Vec<Var>,
858  pub output: Option<KindAnnotation>,
859  pub states: Vec<StateDefinition>,
860}
861
862#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
863#[derive(Clone, Debug, Hash, PartialEq, Eq)]
864pub struct StateDefinition {
865  pub name: Identifier,
866  pub state_variables: Option<Vec<Var>>,
867}
868
869#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
870#[derive(Clone, Debug, Hash, PartialEq, Eq)]
871pub enum Statement {
872  EnumDefine(EnumDefine),
873  //FsmDeclare(FsmDeclare),
874  KindDefine(KindDefine),
875  OpAssign(OpAssign),
876  VariableAssign(VariableAssign),
877  VariableDefine(VariableDefine),
878  TupleDestructure(TupleDestructure),
879  SplitTable,     // todo
880  FlattenTable,   // todo
881}
882
883impl Statement {
884  pub fn tokens(&self) -> Vec<Token> {
885    match self {
886      Statement::EnumDefine(x) => x.tokens(),
887      //Statement::FsmDeclare(x) => x.tokens(),
888      Statement::KindDefine(x) => x.tokens(),
889      Statement::OpAssign(x) => x.tokens(),
890      Statement::VariableAssign(x) => x.tokens(),
891      Statement::VariableDefine(x) => x.tokens(),
892      Statement::TupleDestructure(x) => x.tokens(),
893      Statement::SplitTable => vec![], // todo
894      Statement::FlattenTable => vec![], // todo
895    }
896  }
897}
898
899#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
900#[derive(Clone, Debug, Hash, PartialEq, Eq)]
901pub struct TupleDestructure {
902  pub vars: Vec<Identifier>,
903  pub expression: Expression,
904}
905
906impl TupleDestructure {
907  pub fn tokens(&self) -> Vec<Token> {
908    let mut tokens = vec![];
909    for var in &self.vars {
910      tokens.append(&mut var.tokens());
911    }
912    tokens.append(&mut self.expression.tokens());
913    tokens
914  }
915}
916
917#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
918#[derive(Clone, Debug, Hash, PartialEq, Eq)]
919pub struct FsmPipe {
920  pub start: FsmInstance,
921  pub transitions: Vec<Transition>
922}
923
924#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
925#[derive(Clone, Debug, Hash, PartialEq, Eq)]
926pub enum PipeElement {
927  Expression(Expression),
928  FsmInstance(FsmInstance),
929  Timer // todo
930}
931
932#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
933#[derive(Clone, Debug, Hash, PartialEq, Eq)]
934pub struct FsmDeclare {
935  pub fsm: Fsm,
936  pub pipe: FsmPipe,
937}
938
939#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
940#[derive(Clone, Debug, Hash, PartialEq, Eq)]
941pub struct Fsm {
942  pub name: Identifier,
943  pub args: Option<ArgumentList>,
944  pub kind: Option<KindAnnotation>
945}
946
947pub type FsmArgs = Vec<(Option<Identifier>,Expression)>;
948
949#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
950#[derive(Clone, Debug, Hash, PartialEq, Eq)]
951pub struct FsmInstance {
952  pub name: Identifier,
953  pub args: Option<FsmArgs>,
954}
955
956#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
957#[derive(Clone, Debug, Hash, PartialEq, Eq)]
958pub struct EnumDefine {
959  pub name: Identifier,
960  pub variants: Vec<EnumVariant>,
961}
962
963impl EnumDefine {
964  pub fn tokens(&self) -> Vec<Token> {
965    let mut tokens = self.name.tokens();
966    for v in &self.variants {
967      tokens.append(&mut v.tokens());
968    }
969    tokens
970  }
971}
972
973#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
974#[derive(Clone, Debug, Hash, PartialEq, Eq)]
975pub struct EnumVariant {
976  pub name: Identifier,
977  pub value: Option<KindAnnotation>,
978}
979
980impl EnumVariant {
981  pub fn tokens(&self) -> Vec<Token> {
982    let mut tokens = self.name.tokens();
983    if let Some(value) = &self.value {
984      tokens.append(&mut value.tokens());
985    }
986    tokens
987  }
988}
989
990#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
991#[derive(Clone, Debug, Hash, PartialEq, Eq)]
992pub struct KindDefine {
993  pub name: Identifier,
994  pub kind: KindAnnotation,
995}
996
997impl KindDefine {
998  pub fn tokens(&self) -> Vec<Token> {
999    let mut tokens = self.name.tokens();
1000    tokens.append(&mut self.kind.tokens());
1001    tokens
1002  }
1003}
1004
1005#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1006#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1007pub struct Record {
1008  pub bindings: Vec<Binding>,
1009}
1010
1011#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1012#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1013pub enum Structure {
1014  Empty,
1015  Map(Map),
1016  Matrix(Matrix),
1017  Record(Record),
1018  Set(Set),
1019  Table(Table),
1020  Tuple(Tuple),
1021  TupleStruct(TupleStruct),
1022}
1023
1024impl Structure {
1025  pub fn tokens(&self) -> Vec<Token> {
1026    match self {
1027      Structure::Matrix(mat) => mat.tokens(),
1028      _ => todo!(),
1029    }
1030  }
1031}
1032
1033#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1034#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1035pub struct Map {
1036  pub elements: Vec<Mapping>,
1037}
1038
1039#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1040#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1041pub struct Mapping {
1042  pub key: Expression,
1043  pub value: Expression,
1044}
1045
1046#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1047#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1048pub struct Set {
1049  pub elements: Vec<Expression>,
1050}
1051
1052#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1053#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1054pub struct Atom {
1055  pub name: Identifier,
1056}
1057
1058#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1059#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1060pub struct TupleStruct {
1061  pub name: Identifier,
1062  pub value: Box<Expression>,
1063}
1064
1065#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1066#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1067pub struct Matrix {
1068  pub rows: Vec<MatrixRow>,
1069}
1070
1071impl Matrix {
1072  pub fn tokens(&self) -> Vec<Token> {
1073    let mut tkns = vec![];
1074    for r in &self.rows {
1075      let mut t = r.tokens();
1076      tkns.append(&mut t);
1077    }
1078    tkns
1079  }
1080}
1081
1082pub type TableHeader = Vec<Field>;
1083
1084#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1085#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1086pub struct Table {
1087  pub header: TableHeader,
1088  pub rows: Vec<TableRow>,
1089}
1090
1091#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1092#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1093pub struct Field {
1094  pub name: Identifier,
1095  pub kind: Option<KindAnnotation>,
1096}
1097
1098#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1099#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1100pub struct TableColumn {
1101  pub element: Expression,
1102}
1103
1104#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1105#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1106pub struct MatrixColumn {
1107  pub element: Expression,
1108}
1109
1110impl MatrixColumn {
1111  pub fn tokens(&self) -> Vec<Token> {
1112    self.element.tokens()
1113  }
1114}
1115
1116#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1117#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1118pub struct TableRow {
1119  pub columns: Vec<TableColumn>,
1120}
1121
1122#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1123#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1124pub struct MatrixRow {
1125  pub columns: Vec<MatrixColumn>,
1126}
1127
1128impl MatrixRow {
1129  pub fn tokens(&self) -> Vec<Token> {
1130    let mut tkns = vec![];
1131    for r in &self.columns {
1132      let mut t = r.tokens();
1133      tkns.append(&mut t);
1134    }
1135    tkns
1136  }
1137}
1138
1139#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1140#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1141pub struct VariableDefine {
1142  pub mutable: bool,
1143  pub var: Var,
1144  pub expression: Expression,
1145}
1146
1147impl VariableDefine {
1148  pub fn tokens(&self) -> Vec<Token> {
1149    let mut tkns = self.var.tokens();
1150    tkns.append(&mut self.expression.tokens());
1151    tkns
1152  }
1153}
1154
1155#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1156#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1157pub struct Var {
1158  pub name: Identifier,
1159  pub kind: Option<KindAnnotation>,
1160}
1161
1162impl Var {
1163  pub fn tokens(&self) -> Vec<Token> {
1164    let mut tkns = self.name.tokens();
1165    if let Some(knd) = &self.kind {
1166      let mut t = knd.tokens();
1167      tkns.append(&mut t);
1168    }
1169    tkns
1170  }
1171}
1172
1173
1174#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1175#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1176pub struct VariableAssign {
1177  pub target: SliceRef,
1178  pub expression: Expression,
1179}
1180
1181impl VariableAssign {
1182  pub fn tokens(&self) -> Vec<Token> {
1183    let mut tkns = self.target.tokens();
1184    tkns.append(&mut self.expression.tokens());
1185    tkns
1186  }
1187}
1188
1189#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1190#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1191pub struct Identifier {
1192  pub name: Token,
1193}
1194
1195impl Identifier {
1196  pub fn tokens(&self) -> Vec<Token> {
1197    vec![self.name.clone()]
1198  }
1199
1200  pub fn to_string(&self) -> String {
1201    self.name.to_string()
1202  }
1203
1204}
1205
1206
1207impl Identifier {
1208  pub fn hash(&self) -> u64 {
1209    hash_chars(&self.name.chars)
1210  }
1211}
1212
1213#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1214#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1215pub struct Emoji {
1216  pub tokens: Vec<Token>,
1217}
1218
1219#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1220#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1221pub struct Word {
1222  pub tokens: Vec<Token>,
1223}
1224
1225#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1226#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1227pub struct Slice {
1228  pub name: Identifier,
1229  pub subscript: Vec<Subscript>
1230}
1231
1232impl Slice {
1233  pub fn tokens(&self) -> Vec<Token> {
1234    let mut tkns = self.name.tokens();
1235    for sub in &self.subscript {
1236      let mut sub_tkns = sub.tokens();
1237      tkns.append(&mut sub_tkns);
1238    }
1239    tkns
1240  }
1241}
1242
1243#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1244#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1245pub struct SliceRef {
1246  pub name: Identifier,
1247  pub subscript: Option<Vec<Subscript>>
1248}
1249
1250impl SliceRef {
1251  pub fn tokens(&self) -> Vec<Token> {
1252    let mut tkns = self.name.tokens();
1253    if let Some(subs) = &self.subscript {
1254      for sub in subs {
1255        let mut sub_tkns = sub.tokens();
1256        tkns.append(&mut sub_tkns);
1257      }
1258    }
1259    tkns
1260  }
1261}
1262
1263#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1264#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1265pub enum Subscript {
1266  All,                      // a[:]
1267  Brace(Vec<Subscript>),    // a{"foo"}
1268  Bracket(Vec<Subscript>),  // a[1,2,3]
1269  Dot(Identifier),          // a.b
1270  DotInt(RealNumber),       // a.1
1271  Formula(Factor),          // a[1 + 1]
1272  Range(RangeExpression),   // a[1 + 1]
1273  Swizzle(Vec<Identifier>), // a.b,c
1274}
1275
1276impl Subscript {
1277
1278  pub fn tokens(&self) -> Vec<Token> {
1279    match self {
1280      Subscript::All => vec![
1281        Token::new(TokenKind::Colon, SourceRange::default(), vec![':']),
1282      ],
1283      Subscript::Brace(subs) => {
1284        let mut tkns = vec![Token::new(TokenKind::LeftBrace, SourceRange::default(), vec!['{'])];
1285        for sub in subs {
1286          let mut sub_tkns = sub.tokens();
1287          tkns.append(&mut sub_tkns);
1288        }
1289        tkns.push(Token::new(TokenKind::RightBrace, SourceRange::default(), vec!['}']));
1290        tkns
1291      },
1292      Subscript::Bracket(subs) => {
1293        let mut tkns = vec![Token::new(TokenKind::LeftBracket, SourceRange::default(), vec!['['])];
1294        for sub in subs {
1295          let mut sub_tkns = sub.tokens();
1296          tkns.append(&mut sub_tkns);
1297        }
1298        tkns.push(Token::new(TokenKind::RightBracket, SourceRange::default(), vec![']']));
1299        tkns
1300      },
1301      Subscript::Dot(id) => id.tokens(),
1302      Subscript::DotInt(num) => num.tokens(),
1303      Subscript::Formula(factor) => factor.tokens(),
1304      Subscript::Range(range) => range.tokens(),
1305      Subscript::Swizzle(ids) => ids.iter().flat_map(|id| id.tokens()).collect(),
1306    }
1307  }
1308}
1309
1310#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1311#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1312pub enum Expression {
1313  Formula(Factor),
1314  FunctionCall(FunctionCall),
1315  //FsmPipe(FsmPipe),
1316  Literal(Literal),
1317  Range(Box<RangeExpression>),
1318  Slice(Slice),
1319  Structure(Structure),
1320  Var(Var),
1321}
1322
1323impl Expression {
1324  pub fn tokens(&self) -> Vec<Token> {
1325    match self {
1326      Expression::Var(v) => v.tokens(),
1327      Expression::Literal(ltrl) => ltrl.tokens(),
1328      Expression::Structure(strct) => strct.tokens(),
1329      Expression::Formula(fctr) => fctr.tokens(),
1330      Expression::Range(range) => range.tokens(),
1331      Expression::Slice(slice) => slice.tokens(),
1332      _ => todo!(),
1333    }
1334  }
1335}
1336
1337pub type ArgumentList = Vec<(Option<Identifier>,Expression)>;
1338
1339#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1340#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1341pub struct FunctionCall {
1342  pub name: Identifier,
1343  pub args: ArgumentList,
1344}
1345
1346impl FunctionCall {
1347  pub fn tokens(&self) -> Vec<Token> {
1348    self.name.tokens()
1349  }
1350}
1351
1352#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1353#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1354pub struct Tuple {
1355  pub elements: Vec<Expression>
1356}
1357
1358#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1359#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1360pub struct Binding {
1361  pub name: Identifier,
1362  pub kind: Option<KindAnnotation>,
1363  pub value: Expression,
1364}
1365
1366#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1367#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1368pub struct KindAnnotation {
1369  pub kind: Kind,
1370}
1371
1372impl KindAnnotation {
1373
1374  pub fn hash(&self) -> u64 {
1375    #[cfg(feature = "no_std")]
1376    let mut hasher = FxHasher::default();
1377    #[cfg(not(feature = "no_std"))]
1378    let mut hasher = std::collections::hash_map::DefaultHasher::new();
1379    self.kind.hash(&mut hasher);
1380    hasher.finish()
1381  }
1382
1383  pub fn tokens(&self) -> Vec<Token> {
1384    self.kind.tokens()
1385  }
1386}
1387
1388#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1389#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1390pub enum Kind {
1391  Any,
1392  Atom(Identifier),
1393  Table((Vec<(Identifier,Kind)>,Box<Literal>)),
1394  Set(Box<Kind>,Option<Box<Literal>>),
1395  Record((Vec<(Identifier,Kind)>)),
1396  Empty,
1397  //Fsm(Vec<Kind>,Vec<Kind>),
1398  //Function(Vec<Kind>,Vec<Kind>),
1399  Map(Box<Kind>,Box<Kind>),
1400  Matrix((Box<Kind>,Vec<Literal>)),
1401  Option(Box<Kind>),
1402  Scalar(Identifier),
1403  Tuple(Vec<Kind>),
1404}
1405
1406impl Kind {
1407  pub fn tokens(&self) -> Vec<Token> {
1408    match self {
1409      Kind::Option(x) => x.tokens(),
1410      Kind::Tuple(x) => x.iter().flat_map(|k| k.tokens()).collect(),
1411      Kind::Matrix((kind, literals)) => {
1412        let mut tokens = kind.tokens();
1413        for l in literals {
1414          tokens.append(&mut l.tokens());
1415        }
1416        tokens
1417      },
1418      Kind::Record(kinds) => {
1419        let mut tokens = vec![];
1420        for (id, kind) in kinds {
1421          tokens.append(&mut id.tokens());
1422          tokens.append(&mut kind.tokens());
1423        }
1424        tokens
1425      }
1426      Kind::Set(kind, size) => {
1427        let mut tokens = kind.tokens();
1428        if let Some(literal) = size {
1429          tokens.append(&mut literal.tokens());
1430        }
1431        tokens
1432      }
1433      Kind::Table((kinds, literal)) => {
1434        let mut tokens = vec![];
1435        for (id, kind) in kinds {
1436          tokens.append(&mut id.tokens());
1437          tokens.append(&mut kind.tokens());
1438        }
1439        tokens.append(&mut literal.tokens());
1440        tokens
1441      }
1442      Kind::Map(x, y) => x.tokens().into_iter().chain(y.tokens()).collect(),
1443      Kind::Scalar(x) => x.tokens(),
1444      Kind::Atom(x) => x.tokens(),
1445      Kind::Empty => vec![],
1446      Kind::Any => vec![],
1447    }
1448  }
1449}
1450
1451#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1452#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1453pub enum Literal {
1454  Atom(Atom),
1455  Boolean(Token),
1456  Empty(Token),
1457  Number(Number),
1458  String(MechString),
1459  Kind(Kind),
1460  TypedLiteral((Box<Literal>,KindAnnotation))
1461}
1462
1463impl Literal {
1464  pub fn tokens(&self) -> Vec<Token> {
1465    match self {
1466      Literal::Atom(atm) => atm.name.tokens(),
1467      Literal::Boolean(tkn) => vec![tkn.clone()],
1468      Literal::Number(x) => x.tokens(),
1469      Literal::String(strng) => vec![strng.text.clone()],
1470      Literal::Empty(tkn) => vec![tkn.clone()],
1471      Literal::Kind(knd) => knd.tokens(),
1472      Literal::TypedLiteral((lit, knd)) => {
1473        let mut tokens = lit.tokens();
1474        tokens.append(&mut knd.tokens());
1475        tokens
1476      }
1477    }
1478  }
1479}
1480
1481#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1482#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1483pub struct MechString {
1484  pub text: Token,
1485}
1486
1487impl MechString {
1488  pub fn to_string(&self) -> String {
1489    self.text.to_string()
1490  }
1491}
1492
1493pub type Hyperlink = (Paragraph, Token);
1494
1495#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1496#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1497pub enum ParagraphElement {
1498  Emphasis(Box<ParagraphElement>),
1499  FootnoteReference(Token),
1500  Highlight(Box<ParagraphElement>),
1501  Hyperlink(Hyperlink),
1502  InlineCode(Token),
1503  EvalInlineMechCode(Expression),
1504  InlineMechCode(MechCode),
1505  InlineEquation(Token),
1506  Reference(Token),
1507  SectionReference(Token),
1508  Strikethrough(Box<ParagraphElement>),
1509  Strong(Box<ParagraphElement>),
1510  Text(Token),
1511  Underline(Box<ParagraphElement>),
1512  Error(Token, SourceRange),
1513}
1514
1515impl Recoverable for ParagraphElement {
1516  fn error_placeholder(skipped_tokens: Token, range: SourceRange) -> Self {
1517    ParagraphElement::Error(skipped_tokens, range)
1518  }
1519}
1520
1521impl ParagraphElement {
1522
1523  pub fn tokens(&self) -> Vec<Token> {
1524    match self {
1525      ParagraphElement::Emphasis(t) => t.tokens(),
1526      ParagraphElement::FootnoteReference(t) => vec![t.clone()],
1527      ParagraphElement::Highlight(t) => t.tokens(),
1528      ParagraphElement::Hyperlink((t, u)) => {
1529        let mut tokens = t.tokens();
1530        tokens.push(u.clone());
1531        tokens
1532      },
1533      ParagraphElement::InlineCode(t) => vec![t.clone()],
1534      ParagraphElement::InlineEquation(t) => vec![t.clone()],
1535      ParagraphElement::InlineMechCode(t) => t.tokens(),
1536      ParagraphElement::EvalInlineMechCode(t) => t.tokens(),
1537      ParagraphElement::Reference(t) => vec![t.clone()],
1538      ParagraphElement::SectionReference(t) => vec![t.clone()],
1539      ParagraphElement::Strikethrough(t) => t.tokens(),
1540      ParagraphElement::Strong(t) => t.tokens(),
1541      ParagraphElement::Text(t) => vec![t.clone()],
1542      ParagraphElement::Underline(t) => t.tokens(),
1543      ParagraphElement::Error(t, _) => vec![t.clone()],
1544    }
1545  }
1546
1547  pub fn to_string(&self) -> String {
1548    match self {
1549      ParagraphElement::Emphasis(t) => t.to_string(),
1550      ParagraphElement::FootnoteReference(t) => t.to_string(),
1551      ParagraphElement::Highlight(t) => t.to_string(),
1552      ParagraphElement::Hyperlink((t, u)) => {
1553        format!("[{}]({})", t.to_string(), u.to_string())
1554      }
1555      ParagraphElement::InlineCode(t) => t.to_string(),
1556      ParagraphElement::InlineEquation(t) => t.to_string(),
1557      ParagraphElement::InlineMechCode(t) => format!("{:?}", t),
1558      ParagraphElement::EvalInlineMechCode(t) => format!("{:?}", t),
1559      ParagraphElement::Reference(t) => t.to_string(),
1560      ParagraphElement::SectionReference(t) => t.to_string(),
1561      ParagraphElement::Strikethrough(t) => t.to_string(),
1562      ParagraphElement::Strong(t) => t.to_string(),
1563      ParagraphElement::Text(t) => t.to_string(),
1564      ParagraphElement::Underline(t) => t.to_string(),
1565      ParagraphElement::Error(t, s) => format!("{{ERROR: {} at {:?}}}", t.to_string(), s),
1566    }
1567  }
1568
1569}
1570
1571#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1572#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1573pub struct Paragraph {
1574  pub elements: Vec<ParagraphElement>,
1575  pub error_range: Option<SourceRange>,
1576}
1577
1578impl Paragraph {
1579  pub fn to_string(&self) -> String {
1580    let mut out = "".to_string();
1581    for e in &self.elements {
1582      out.push_str(&e.to_string());
1583    }
1584    out
1585  }
1586
1587  pub fn from_tokens(tokens: Vec<Token>) -> Paragraph {
1588    let elements = tokens.into_iter().map(|t| ParagraphElement::Text(t)).collect();
1589    Paragraph {
1590      elements,
1591      error_range: None,
1592    }
1593  }
1594
1595  pub fn has_errors(&self) -> bool {
1596    self.error_range.is_some()
1597  }
1598
1599  pub fn tokens(&self) -> Vec<Token> {
1600    let mut tkns = vec![];
1601    for e in &self.elements {
1602      let mut e_tkns = e.tokens();
1603      tkns.append(&mut e_tkns);
1604    }
1605    tkns
1606  }
1607
1608}
1609
1610impl Recoverable for Paragraph {
1611  fn error_placeholder(skipped_tokens: Token, range: SourceRange) -> Self {
1612    Paragraph {
1613      elements: vec![ParagraphElement::Text(Token::new(TokenKind::Error, range.clone(), skipped_tokens.chars.clone()))],
1614      error_range: Some(range),
1615    }
1616  }
1617}
1618
1619
1620pub type Sign = bool;
1621pub type Numerator = Token;
1622pub type Denominator = Token;
1623pub type Whole = Token;
1624pub type Part = Token;
1625pub type Real = Box<Number>;
1626pub type Imaginary = Box<Number>;
1627pub type Base = (Whole, Part);
1628pub type Exponent = (Sign, Whole, Part);
1629
1630#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1631#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1632pub enum Number {
1633  Real(RealNumber),
1634  Complex(C64Node),
1635}
1636
1637impl Number {
1638
1639  pub fn from_integer(x: i64) -> Number {
1640    Number::Real(RealNumber::Integer(Token::new(TokenKind::Digit, SourceRange::default(), x.to_string().chars().collect())))
1641  }
1642
1643  pub fn to_string(&self) -> String {
1644    match self {
1645      Number::Real(x) => x.to_string(),
1646      Number::Complex(x) => x.to_string(),
1647    }
1648  }
1649
1650  pub fn tokens(&self) -> Vec<Token> {
1651    match self {
1652      Number::Real(x) => x.tokens(),
1653      Number::Complex(x) => x.tokens(),
1654    }
1655  }
1656}
1657
1658#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1659#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1660pub enum RealNumber {
1661  Binary(Token),
1662  Decimal(Token),
1663  Float((Whole,Part)),
1664  Hexadecimal(Token),
1665  Integer(Token),
1666  Negated(Box<RealNumber>),
1667  Octal(Token),
1668  Rational((Numerator,Denominator)),
1669  Scientific((Base,Exponent)),
1670}
1671
1672impl RealNumber {
1673  pub fn tokens(&self) -> Vec<Token> {
1674    match self {
1675      RealNumber::Integer(tkn) => vec![tkn.clone()],
1676      _ => todo!(),
1677    }
1678  }
1679  pub fn to_string(&self) -> String {
1680    match self {
1681      RealNumber::Integer(tkn) => tkn.to_string(),
1682      RealNumber::Float((whole, part)) => format!("{}.{}", whole.to_string(), part.to_string()),
1683      RealNumber::Binary(tkn) => format!("0b{}", tkn.to_string()),
1684      RealNumber::Hexadecimal(tkn) => format!("0x{}", tkn.to_string()),
1685      RealNumber::Octal(tkn) => format!("0o{}", tkn.to_string()),
1686      RealNumber::Decimal(tkn) => format!("0d{}", tkn.to_string()),
1687      RealNumber::Rational((num, den)) => format!("{}/{}", num.to_string(), den.to_string()),
1688      RealNumber::Scientific(((whole,part), exponent)) => {
1689        let (sign, whole, part) = exponent;
1690        let sign_str = if *sign { "+" } else { "-" };
1691        let whole_str = whole.to_string();
1692        let part_str = part.to_string();
1693        format!("{}{}.{}/10^{}", whole.to_string(), part.to_string(), sign_str, whole_str)
1694      }
1695      RealNumber::Negated(x) => format!("-{}", x.to_string()),
1696    }
1697  }
1698}
1699
1700#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1701#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1702pub struct ImaginaryNumber {
1703  pub number: RealNumber,
1704}
1705
1706#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1707#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1708pub struct C64Node {
1709  pub real: Option<RealNumber>,
1710  pub imaginary: ImaginaryNumber
1711}
1712
1713impl C64Node {
1714  pub fn tokens(&self) -> Vec<Token> {
1715    let mut tkns = vec![];
1716    if let Some(r) = &self.real {
1717      tkns.append(&mut r.tokens());
1718    }
1719    tkns.append(&mut self.imaginary.number.tokens());
1720    tkns
1721  }
1722
1723  pub fn to_string(&self) -> String {
1724    let mut out = "".to_string();
1725    if let Some(r) = &self.real {
1726      out.push_str(&r.to_string());
1727    }
1728    out.push_str("i");
1729    out
1730  }
1731}
1732
1733#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1734#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1735pub struct Comment {
1736  pub paragraph: Paragraph,
1737}
1738
1739impl Comment {
1740  pub fn tokens(&self) -> Vec<Token> {
1741    self.paragraph.tokens()
1742  }
1743}
1744
1745#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1746#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1747pub struct OpAssign {
1748  pub target: SliceRef,
1749  pub op: OpAssignOp,
1750  pub expression: Expression,
1751}
1752
1753impl OpAssign {
1754  pub fn tokens(&self) -> Vec<Token> {
1755    let mut tkns = self.target.tokens();
1756    tkns.append(&mut self.op.tokens());
1757    tkns.append(&mut self.expression.tokens());
1758    tkns
1759  }
1760}
1761
1762#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1763#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1764pub enum OpAssignOp {
1765  Add,
1766  Div,
1767  Exp, 
1768  Mod,  
1769  Mul,
1770  Sub,   
1771}
1772
1773impl OpAssignOp {
1774  pub fn tokens(&self) -> Vec<Token> {
1775    match self {
1776      OpAssignOp::Add => vec![Token::new(TokenKind::Plus, SourceRange::default(), vec!['+'])],
1777      OpAssignOp::Div => vec![Token::new(TokenKind::Slash, SourceRange::default(), vec!['/'])],
1778      OpAssignOp::Exp => vec![Token::new(TokenKind::Caret, SourceRange::default(), vec!['^'])],
1779      OpAssignOp::Mod => vec![Token::new(TokenKind::Percent, SourceRange::default(), vec!['%'])],
1780      OpAssignOp::Mul => vec![Token::new(TokenKind::Asterisk, SourceRange::default(), vec!['*'])],
1781      OpAssignOp::Sub => vec![Token::new(TokenKind::Dash, SourceRange::default(), vec!['-'])],
1782    }
1783  }
1784}
1785
1786#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1787#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1788pub enum RangeOp {
1789  Exclusive,      
1790  Inclusive,
1791}
1792
1793#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1794#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1795pub enum AddSubOp {
1796  Add,
1797  Sub,
1798}
1799
1800#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1801#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1802pub enum MulDivOp {
1803  Div,
1804  Mod,
1805  Mul,
1806}
1807
1808#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1809#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1810pub enum VecOp {
1811  Cross,
1812  Dot,
1813  MatMul,
1814  Solve,
1815}
1816
1817#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1818#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1819pub enum PowerOp {
1820  Pow
1821}
1822
1823#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1824#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1825pub enum ComparisonOp {
1826  Equal,
1827  GreaterThan,
1828  GreaterThanEqual,
1829  LessThan,
1830  LessThanEqual,
1831  NotEqual,
1832  StrictEqual,
1833  StrictNotEqual,
1834}
1835
1836#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1837#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1838pub enum LogicOp {
1839  And,
1840  Not,
1841  Or,
1842  Xor,
1843}
1844
1845#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1846#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1847pub enum FormulaOperator {
1848  AddSub(AddSubOp),
1849  Comparison(ComparisonOp),
1850  Power(PowerOp),
1851  Logic(LogicOp),
1852  MulDiv(MulDivOp),
1853  Vec(VecOp),
1854  Table(TableOp),
1855  Set(SetOp),
1856}
1857
1858#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1859#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1860pub enum TableOp {
1861  InnerJoin,
1862  LeftOuterJoin,
1863  RightOuterJoin	,
1864  FullOuterJoin	,
1865  LeftSemiJoin,  
1866  LeftAntiJoin,
1867}
1868
1869#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1870#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1871pub enum SetOp {
1872  Union,
1873  Intersection,
1874  Difference,
1875  Complement,
1876  ElementOf,
1877  NotElementOf,
1878  ProperSubset,
1879  ProperSuperset,
1880  Subset,
1881  Superset,
1882  SymmetricDifference,
1883}
1884
1885#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1886#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1887pub struct RangeExpression {
1888  pub start: Factor,
1889  pub increment: Option<(RangeOp,Factor)>,
1890  pub operator: RangeOp,
1891  pub terminal: Factor,
1892}
1893
1894impl RangeExpression {
1895  pub fn tokens(&self) -> Vec<Token> {
1896    let mut tokens = self.start.tokens();
1897    tokens.append(&mut self.terminal.tokens());
1898    tokens
1899  }
1900}
1901
1902#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1903#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1904pub struct Term {
1905  pub lhs: Factor,
1906  pub rhs: Vec<(FormulaOperator,Factor)>
1907}
1908
1909impl Term {
1910  pub fn tokens(&self) -> Vec<Token> {
1911    let mut lhs_tkns = self.lhs.tokens();
1912    let mut rhs_tkns = vec![];
1913    for (op, r) in &self.rhs {
1914      let mut tkns = r.tokens();
1915      rhs_tkns.append(&mut tkns);
1916    }
1917    lhs_tkns.append(&mut rhs_tkns);
1918    lhs_tkns
1919  }
1920}
1921
1922#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
1923#[derive(Clone, Debug, Hash, PartialEq, Eq)]
1924pub enum Factor {
1925  Expression(Box<Expression>),
1926  Negate(Box<Factor>),
1927  Not(Box<Factor>),
1928  Parenthetical(Box<Factor>),
1929  Term(Box<Term>),
1930  Transpose(Box<Factor>),
1931}
1932
1933impl Factor {
1934  pub fn tokens(&self) -> Vec<Token> {
1935    match self {
1936      Factor::Expression(x) => x.tokens(),
1937      Factor::Negate(x) => x.tokens(),
1938      Factor::Not(x) => x.tokens(),
1939      Factor::Parenthetical(x) => x.tokens(),
1940      Factor::Term(x) => x.tokens(),
1941      Factor::Transpose(x) => x.tokens(),
1942    }
1943  }
1944}
mech_core/nodes.rs

mech_core/
nodes.rs
