nadesiko3/
token.rs

1//! トークンを定義したもの
2
3use crate::nvalue::NValue;
4
5/// トークンの一覧
6#[derive(Debug,Clone,PartialEq,Copy)]
7pub enum TokenKind {
8    None,
9    Comment,
10    Comma,
11    Eol,
12    Int,
13    Number,
14    String,
15    Word,
16    Flag,
17    Eq,
18    NotEq,
19    Gt,
20    GtEq,
21    Lt,
22    LtEq,
23    Not,
24    ParenL,
25    ParenR,
26    BracketL,
27    BracketR,
28    CurBracketL,
29    CurBracketR,
30    BlockBegin,
31    BlockEnd,
32    If,
33    Else,
34    Kai,
35    Plus,
36    Minus,
37    Mul,
38    Div,
39    Mod,
40    Pow,
41    PlusStr,
42    And,
43    Or,
44    True,
45    False,
46    Break,
47    Continue,
48    For,
49    DefFunc,
50    Return,
51    DefVar,
52    Dainyu,
53}
54
55/// トークンのソースコード情報を表現する構造体
56#[derive(Debug,Clone,Copy,PartialEq)]
57pub struct TokenPos {
58    pub start: i32,
59    pub end: i32,
60    pub fileno: i32,
61    pub row: i32, // 字句解析した後で設定する
62    pub col: i32, // 字句解析した後で設定する
63}
64
65impl TokenPos {
66    pub fn new(start: i32, end: i32, fileno: i32) -> Self {
67        Self { start, end, fileno, row: 0, col: 0 }
68    }
69    pub fn to_string_se(&self) -> String {
70        format!("({},{})", self.start, self.end)
71    }
72}
73
74/// トークンを表現する構造体
75#[derive(Debug,Clone)]
76pub struct Token {
77    pub kind: TokenKind,
78    pub value: NValue,
79    pub josi: Option<String>,
80    pub pos: TokenPos,
81}
82
83impl Token {
84    /// new token
85    pub fn new(kind: TokenKind, value:NValue, josi: Option<String>, pos: TokenPos) -> Self {
86        Self { kind, value, josi, pos }
87    }
88    /// new empty token
89    pub fn new_empty() -> Self {
90        Self::new(TokenKind::None, NValue::Empty, None, TokenPos::new(0, 0, 0))
91    }
92    /// new comment token
93    pub fn new_comment(comment: &str, pos: TokenPos) -> Self {
94        Self::new(TokenKind::Comment, NValue::from_str(comment), None, pos)
95    }
96    /// new token form char
97    pub fn new_char(kind: TokenKind, label: char, pos: TokenPos) -> Self {
98        Self {
99            kind,
100            value: NValue::from_char(label),
101            josi: None,
102            pos,
103        }
104    }
105    /// new token from string
106    pub fn new_str(kind: TokenKind, label: &str, pos: TokenPos) -> Self {
107        Self {
108            kind,
109            value: NValue::from_str(label),
110            josi: None,
111            pos,
112        }
113    }
114    pub fn as_char(&self) -> char {
115        match &self.value {
116            NValue::String(c) => if c.len() > 0 { c.chars().nth(0).unwrap_or('\0') } else { '\0' },
117            _ => '\0',
118        }
119    }
120    pub fn as_label(&self) -> String {
121        self.value.to_string()
122    }
123}
124
125impl std::fmt::Display for Token {
126    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
127        // 助詞の有無に生じて出力方式を変更する
128        let get_value = |t: &Token| -> String {
129            match &t.josi {
130                Some(j) => { format!("{}/{}", t.value.to_string(), j) },
131                None    => { format!("{}", t.value.to_string()) },
132            }
133        };
134        let t = &self;
135        let s: String = match self.kind {
136            TokenKind::None => format!("None"),
137            TokenKind::Comment => format!("Comment:{}", get_value(t)),
138            TokenKind::Comma => format!(","),
139            TokenKind::Eol => format!("Eol"),
140            TokenKind::Int => format!("Int:{}", get_value(t)),
141            TokenKind::Number => format!("Number:{}", get_value(t)),
142            TokenKind::String => format!("String:{}", get_value(t)),
143            TokenKind::Word => format!("Word:{}", get_value(t)),
144            TokenKind::Flag => format!("Flag:{}", get_value(t)),
145            TokenKind::ParenL => String::from("("),
146            TokenKind::ParenR => String::from(")"),
147            TokenKind::Eq => format!("="),
148            TokenKind::NotEq => format!("≠"),
149            TokenKind::Plus => format!("+"),
150            TokenKind::Minus => format!("-"),
151            TokenKind::Mul => format!("*"),
152            TokenKind::Div => format!("/"),
153            TokenKind::Mod => format!("%"),
154            TokenKind::Pow => format!("^"),
155            TokenKind::Gt => format!(">"),
156            TokenKind::GtEq => format!("≧"),
157            TokenKind::Lt => format!("<"),
158            TokenKind::LtEq => format!("≦"),
159            TokenKind::Not => format!("!"),
160            TokenKind::If => format!("もし"),
161            TokenKind::Else => format!("違えば"),
162            TokenKind::Kai => format!("Kai"),
163            TokenKind::BlockBegin => format!("ここから"),
164            TokenKind::BlockEnd => format!("ここまで"),
165            TokenKind::BracketL => String::from("["),
166            TokenKind::BracketR => String::from("]"),
167            TokenKind::CurBracketL => String::from("{"),
168            TokenKind::CurBracketR => String::from("}"),
169            TokenKind::True => String::from("真"),
170            TokenKind::False => String::from("偽"),
171            TokenKind::And => String::from("&&"),
172            TokenKind::Or => String::from("||"),
173            TokenKind::PlusStr => String::from("&"),
174            TokenKind::Break => String::from("抜"),
175            TokenKind::Continue => String::from("続"),
176            TokenKind::For => String::from("繰返"),
177            TokenKind::DefFunc => String::from("●関数"),
178            TokenKind::Return => String::from("戻"),
179            TokenKind::DefVar => String::from("変数"),
180            TokenKind::Dainyu => String::from("代入"),
181            // _ => format!("{:?}", self),
182        };
183        write!(f, "{}", s)
184    }
185}
186
187#[allow(dead_code)]
188pub fn tokens_string(vt: &[Token]) -> String {
189    let mut res = String::new();
190    for tok in vt.iter() {
191        let s = format!("[{}]", tok);
192        res.push_str(&s);
193    }
194    format!("{}", res)
195}
196
197#[allow(dead_code)]
198pub fn tokens_string_pos(vt: &[Token]) -> String {
199    let mut res = String::new();
200    for tok in vt.iter() {
201        let s = format!("[{}]{}", tok, tok.pos.to_string_se());
202        res.push_str(&s);
203    }
204    format!("{}", res)
205}
206
207#[allow(dead_code)]
208pub fn tokens_string_lineno(vt: &[Token]) -> String {
209    let mut res = String::new();
210    for tok in vt.iter() {
211        let s = format!("[{}]({})", tok, tok.pos.row);
212        res.push_str(&s);
213    }
214    format!("{}", res)
215}
nadesiko3/token.rs

nadesiko3/
token.rs
