1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
using System;
using Antlr4.Runtime;
using Antlr4.Runtime.Misc;
using Antlr4.Runtime.Sharpen;
namespace Antlr4.Runtime
{
/// <summary>
/// This default implementation of
/// <see cref="ITokenFactory"/>
/// creates
/// <see cref="CommonToken"/>
/// objects.
/// </summary>
public class CommonTokenFactory : ITokenFactory
{
/// <summary>
/// The default
/// <see cref="CommonTokenFactory"/>
/// instance.
/// <p>
/// This token factory does not explicitly copy token text when constructing
/// tokens.</p>
/// </summary>
public static readonly ITokenFactory Default = new Antlr4.Runtime.CommonTokenFactory();
/// <summary>
/// Indicates whether
/// <see cref="CommonToken.Text"/>
/// should be called after
/// constructing tokens to explicitly set the text. This is useful for cases
/// where the input stream might not be able to provide arbitrary substrings
/// of text from the input after the lexer creates a token (e.g. the
/// implementation of
/// <see cref="ICharStream.GetText(Antlr4.Runtime.Misc.Interval)"/>
/// in
/// <see cref="UnbufferedCharStream"/>
/// throws an
/// <see cref="System.NotSupportedException"/>
/// ). Explicitly setting the token text
/// allows
/// <see cref="IToken.Text()"/>
/// to be called at any time regardless of the
/// input stream implementation.
/// <p>
/// The default value is
/// <see langword="false"/>
/// to avoid the performance and memory
/// overhead of copying text for every token unless explicitly requested.</p>
/// </summary>
protected internal readonly bool copyText;
/// <summary>
/// Constructs a
/// <see cref="CommonTokenFactory"/>
/// with the specified value for
/// <see cref="copyText"/>
/// .
/// <p>
/// When
/// <paramref name="copyText"/>
/// is
/// <see langword="false"/>
/// , the
/// <see cref="Default"/>
/// instance
/// should be used instead of constructing a new instance.</p>
/// </summary>
/// <param name="copyText">
/// The value for
/// <see cref="copyText"/>
/// .
/// </param>
public CommonTokenFactory(bool copyText)
{
this.copyText = copyText;
}
/// <summary>
/// Constructs a
/// <see cref="CommonTokenFactory"/>
/// with
/// <see cref="copyText"/>
/// set to
/// <see langword="false"/>
/// .
/// <p>
/// The
/// <see cref="Default"/>
/// instance should be used instead of calling this
/// directly.</p>
/// </summary>
public CommonTokenFactory()
: this(false)
{
}
public virtual CommonToken Create(Tuple<ITokenSource, ICharStream> source, int type, string text, int channel, int start, int stop, int line, int charPositionInLine)
{
CommonToken t = new CommonToken(source, type, channel, start, stop);
t.Line = line;
t.Column = charPositionInLine;
if (text != null)
{
t.Text = text;
}
else
{
if (copyText && source.Item2 != null)
{
t.Text = source.Item2.GetText(Interval.Of(start, stop));
}
}
return t;
}
IToken ITokenFactory.Create(Tuple<ITokenSource, ICharStream> source, int type, string text, int channel, int start, int stop, int line, int charPositionInLine)
{
return Create(source, type, text, channel, start, stop, line, charPositionInLine);
}
public virtual CommonToken Create(int type, string text)
{
return new CommonToken(type, text);
}
IToken ITokenFactory.Create(int type, string text)
{
return Create(type, text);
}
}
}