syntree 0.18.0

A memory efficient syntax tree for language developers.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

//! Example that showcases why it might be useful to smuggle more data into the
//! syntax tree.

use std::collections::HashMap;
use std::iter::Peekable;

use anyhow::Result;
use syntree::{print, Builder};

#[derive(Debug, Clone, Copy)]
enum Syntax {
    /// A string referenced somewhere else using the provided ID.
    Synthetic(usize),
    /// A literal string from the source.
    Literal,
    /// Whitespace.
    Whitespace,
    /// A lexer error.
    Error,
}

use Syntax::*;

#[derive(Default)]
struct Storage {
    lookup: HashMap<String, usize>,
    storage: Vec<String>,
}

impl Storage {
    /// Insert a string into synthetic storage.
    fn insert(&mut self, name: &str, value: &str) -> usize {
        let id = self.storage.len();
        self.storage.push(value.to_string());
        self.lookup.insert(name.to_owned(), id);
        id
    }

    /// Lookup a string from synthetic storage by name.
    fn lookup(&self, name: &str) -> Option<usize> {
        self.lookup.get(name).copied()
    }

    fn get(&self, id: usize) -> Option<&str> {
        Some(self.storage.get(id)?.as_str())
    }
}

fn lexer<'a>(source: &'a str, storage: &'a Storage) -> impl Iterator<Item = (Syntax, usize)> + 'a {
    let mut it = source.char_indices().peekable();
    let len = source.len();

    return std::iter::from_fn(move || {
        let (start, c) = it.next()?;

        let syntax = match c {
            c if c.is_whitespace() => {
                eat(&mut it, char::is_whitespace);
                Whitespace
            }
            '$' => {
                eat(&mut it, |c| c.is_ascii_lowercase());
                let end = it.peek().map_or(len, |(n, _)| *n);
                let id = &source[(start + 1)..end];

                match storage.lookup(id) {
                    Some(id) => Synthetic(id),
                    None => Error,
                }
            }
            'A'..='Z' | 'a'..='z' => {
                eat(&mut it, |c| c.is_ascii_alphabetic());
                Literal
            }
            _ => {
                eat(&mut it, |c| !c.is_whitespace());
                Error
            }
        };

        let end = it.peek().map_or(len, |(n, _)| *n);
        Some((syntax, end.saturating_sub(start)))
    });

    /// Consume all available whitespace.
    fn eat(it: &mut Peekable<impl Iterator<Item = (usize, char)>>, cond: fn(char) -> bool) {
        while it.peek().filter(|&(_, c)| cond(*c)).is_some() {
            it.next();
        }
    }
}

fn main() -> Result<()> {
    let source = std::env::args().skip(1).collect::<String>();

    let mut storage = Storage::default();
    storage.insert("world", "Earth");

    let lexer = lexer(&source, &storage);

    let mut tree = Builder::new();

    for (syntax, len) in lexer {
        tree.token(syntax, len)?;
    }

    let tree = tree.build()?;

    println!("Tree:");

    print::print_with_source(std::io::stdout(), &tree, &source)?;

    println!("Eval:");

    let mut count = 0usize;

    for node in tree.children() {
        let string = match node.value() {
            Synthetic(id) => match storage.get(id) {
                Some(string) => string,
                None => {
                    println!("{} = {} (not found)", count, &source[node.range()]);
                    count += 1;
                    continue;
                }
            },
            Literal => &source[node.range()],
            Whitespace => continue,
            Error => {
                println!("Error: {}", &source[node.range()]);
                continue;
            }
        };

        println!("{count} = {string:?}");
        count += 1;
    }

    Ok(())
}