unsynn 0.3.0

(Proc-macro) parsing made easy
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

//! Iterator type for parsing token streams.

#[cfg(feature = "proc_macro2")]
use proc_macro2::{TokenStream, TokenTree};

#[cfg(not(feature = "proc_macro2"))]
use proc_macro::{TokenStream, TokenTree};

use crate::EndOfStream;
use crate::Parser;

/// The base iterator
#[derive(Clone)]
#[cfg_attr(feature = "proc_macro2", derive(Debug))]
struct TokenIterBase {
    /// The actual token iterator
    iter: <TokenStream as IntoIterator>::IntoIter,
    /// Counter for this level
    counter: usize,

    #[cfg(feature = "extra_asserts")]
    /// Tracks if a nested iterator has been committed at the current position
    /// Set to true on commit, cleared when parent advances via `.next()`
    committed: bool,
}

impl TokenIterBase {
    fn new(
        iter: impl IntoIterator<IntoIter = <TokenStream as IntoIterator>::IntoIter>,
        start_at: usize,
    ) -> Self {
        Self {
            iter: iter.into_iter(),
            counter: start_at,
            #[cfg(feature = "extra_asserts")]
            committed: false,
        }
    }
}

/// Iterator type for parsing token streams.
///
/// # Examples
///
/// ```rust
/// # use unsynn::*;
/// // Simple parsing
/// let tokens = "foo bar".to_token_iter();
/// ```
#[derive(Clone)]
#[cfg_attr(feature = "proc_macro2", derive(Debug))]
pub struct TokenIter {
    base: TokenIterBase,
}

impl TokenIter {
    /// Create a new owned `TokenIter`
    pub fn new(
        stream: impl IntoIterator<IntoIter = <TokenStream as IntoIterator>::IntoIter>,
    ) -> Self {
        TokenIter {
            base: TokenIterBase::new(stream, 0),
        }
    }
}

impl TokenIter {
    /// Get the current counter value
    #[must_use]
    pub const fn counter(&self) -> usize {
        self.base.counter
    }

    /// Adjust the counter by an offset
    pub(crate) fn add(&mut self, add: usize) {
        self.base.counter += add;
    }

    /// Scoped parsing of a nested group.
    ///
    /// Creates a new iterator with the stream to the inner of the group,
    /// parses type `T`, and ensures all tokens are consumed.
    pub(crate) fn parse_group<T>(
        &mut self,
        stream: impl IntoIterator<IntoIter = <TokenStream as IntoIterator>::IntoIter>,
    ) -> crate::Result<T>
    where
        T: crate::Parser,
    {
        #[cfg(feature = "extra_asserts")]
        assert!(!self.base.committed);
        let mut tokens = TokenIter {
            base: TokenIterBase::new(stream, self.counter()),
        };
        let parsed = T::parser(&mut tokens)?;
        // Must parse exhaustively, otherwise we would miscount tokens
        EndOfStream::parser(&mut tokens)?;
        self.base.counter = tokens.counter();
        #[cfg(feature = "extra_asserts")]
        {
            self.base.committed = true;
        };
        Ok(parsed)
    }
}

// Iterator implementation
impl Iterator for TokenIter {
    type Item = TokenTree;

    fn next(&mut self) -> Option<Self::Item> {
        let item = self.base.iter.next()?;
        self.base.counter += 1;
        #[cfg(feature = "extra_asserts")]
        {
            self.base.committed = false;
        };
        Some(item)
    }
}

/// Extension trait to convert iterators into [`TokenIter`].
pub trait IntoTokenIter: Sized {
    /// Convert this iterator into a [`TokenIter`].
    fn into_token_iter(self) -> TokenIter;
}

impl IntoTokenIter for <TokenStream as IntoIterator>::IntoIter {
    fn into_token_iter(self) -> TokenIter {
        TokenIter {
            base: TokenIterBase::new(self, 0),
        }
    }
}

/// Extension trait to convert `TokenStreams` into [`TokenIter`].
pub trait ToTokenIter: Sized {
    /// Convert this into a [`TokenIter`].
    fn to_token_iter(self) -> TokenIter;
}

impl ToTokenIter for TokenStream {
    fn to_token_iter(self) -> TokenIter {
        TokenIter::new(self)
    }
}