1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
using Antlr4.Runtime;
using Antlr4.Runtime.Sharpen;
namespace Antlr4.Runtime
{
/// <summary>
/// This class extends
/// <see cref="BufferedTokenStream"/>
/// with functionality to filter
/// token streams to tokens on a particular channel (tokens where
/// <see cref="IToken.Channel()"/>
/// returns a particular value).
/// <p>
/// This token stream provides access to all tokens by index or when calling
/// methods like
/// <see cref="BufferedTokenStream.GetText()"/>
/// . The channel filtering is only used for code
/// accessing tokens via the lookahead methods
/// <see cref="BufferedTokenStream.LA(int)"/>
/// ,
/// <see cref="LT(int)"/>
/// , and
/// <see cref="Lb(int)"/>
/// .</p>
/// <p>
/// By default, tokens are placed on the default channel
/// (
/// <see cref="TokenConstants.DefaultChannel"/>
/// ), but may be reassigned by using the
/// <c>->channel(HIDDEN)</c>
/// lexer command, or by using an embedded action to
/// call
/// <see cref="Lexer.Channel"/>
/// .
/// </p>
/// <p>
/// Note: lexer rules which use the
/// <c>->skip</c>
/// lexer command or call
/// <see cref="Lexer.Skip()"/>
/// do not produce tokens at all, so input text matched by
/// such a rule will not be available as part of the token stream, regardless of
/// channel.</p>
/// </summary>
public class CommonTokenStream : BufferedTokenStream
{
/// <summary>Specifies the channel to use for filtering tokens.</summary>
/// <remarks>
/// Specifies the channel to use for filtering tokens.
/// <p>
/// The default value is
/// <see cref="TokenConstants.DefaultChannel"/>
/// , which matches the
/// default channel assigned to tokens created by the lexer.</p>
/// </remarks>
protected internal int channel = TokenConstants.DefaultChannel;
/// <summary>
/// Constructs a new
/// <see cref="CommonTokenStream"/>
/// using the specified token
/// source and the default token channel (
/// <see cref="TokenConstants.DefaultChannel"/>
/// ).
/// </summary>
/// <param name="tokenSource">The token source.</param>
public CommonTokenStream(ITokenSource tokenSource)
: base(tokenSource)
{
}
/// <summary>
/// Constructs a new
/// <see cref="CommonTokenStream"/>
/// using the specified token
/// source and filtering tokens to the specified channel. Only tokens whose
/// <see cref="IToken.Channel()"/>
/// matches
/// <paramref name="channel"/>
/// or have the
/// <see cref="IToken.Type()"/>
/// equal to
/// <see cref="TokenConstants.EOF"/>
/// will be returned by the
/// token stream lookahead methods.
/// </summary>
/// <param name="tokenSource">The token source.</param>
/// <param name="channel">The channel to use for filtering tokens.</param>
public CommonTokenStream(ITokenSource tokenSource, int channel)
: this(tokenSource)
{
this.channel = channel;
}
protected internal override int AdjustSeekIndex(int i)
{
return NextTokenOnChannel(i, channel);
}
protected internal override IToken Lb(int k)
{
if (k == 0 || (p - k) < 0)
{
return null;
}
int i = p;
int n = 1;
// find k good tokens looking backwards
while (n <= k)
{
// skip off-channel tokens
i = PreviousTokenOnChannel(i - 1, channel);
n++;
}
if (i < 0)
{
return null;
}
return tokens[i];
}
public override IToken LT(int k)
{
//System.out.println("enter LT("+k+")");
LazyInit();
if (k == 0)
{
return null;
}
if (k < 0)
{
return Lb(-k);
}
int i = p;
int n = 1;
// we know tokens[p] is a good one
// find k good tokens
while (n < k)
{
// skip off-channel tokens, but make sure to not look past EOF
if (Sync(i + 1))
{
i = NextTokenOnChannel(i + 1, channel);
}
n++;
}
// if ( i>range ) range = i;
return tokens[i];
}
/// <summary>Count EOF just once.</summary>
/// <remarks>Count EOF just once.</remarks>
public virtual int GetNumberOfOnChannelTokens()
{
int n = 0;
Fill();
for (int i = 0; i < tokens.Count; i++)
{
IToken t = tokens[i];
if (t.Channel == channel)
{
n++;
}
if (t.Type == TokenConstants.EOF)
{
break;
}
}
return n;
}
}
}