1use core::fmt;
2use std::{collections::HashSet, mem::take};
3use linked_hash_map::LinkedHashMap as HashMap;
4
5use rowan::ast::AstNode;
6use to_true::{InTrue, ToTrue};
7use unicode_ident::{is_xid_continue, is_xid_start};
8
9use crate::utils::UsedBound;
10
11mod utils;
12mod bootstarp;
13
14pub use bootstarp::*;
15
16impl Repeat {
17 pub fn count_bounds(&self) -> (u32, Option<u32>) {
18 if self.plus().is_some() {
19 (1, None)
20 } else if let Some(rest) = self.repeat_rest() {
21 let lower_bound = self.number().as_ref().map_or(0, value::number);
22 let upper_bound = rest.number().as_ref().map(value::number);
23 (lower_bound, upper_bound)
24 } else if let Some(number) = self.number() {
25 let bound = value::number(&number);
26 (bound, bound.into())
27 } else {
28 unreachable!()
29 }
30 }
31}
32
33pub mod value {
34 use crate::{SyntaxKind as Kind, Label, SyntaxToken};
35
36 #[track_caller]
37 pub fn string(s: &SyntaxToken) -> &str {
38 debug_assert_eq!(s.kind(), Kind::STRING);
39 let s = s.text();
40 &s[1..s.len()-1]
41 }
42
43 #[track_caller]
44 pub fn matches(s: &SyntaxToken) -> &str {
45 debug_assert_eq!(s.kind(), Kind::MATCHES);
46 let s = s.text();
47 &s[1..s.len()-1]
48 }
49
50 #[track_caller]
51 pub fn label(l: &Label) -> String {
52 l.ident()
53 .map(|ident| ident.text().to_owned())
54 .unwrap_or_else(|| string(&l.string().unwrap()).to_owned())
55 }
56
57 pub fn number(s: &SyntaxToken) -> u32 {
58 debug_assert_eq!(s.kind(), Kind::NUMBER);
59 s.text().parse().unwrap()
60 }
61}
62
63#[derive(Debug)]
64pub enum Error {
65 EmptyLiteral(SyntaxToken),
66 UnknownLiteral(SyntaxToken),
67 MatchesWithoutSlice(SyntaxToken),
68 DisallowedSlice(SyntaxNode),
69}
70
71type Result<T, E = Error> = core::result::Result<T, E>;
72
73#[derive(Debug, PartialEq, Eq)]
74enum Method {
75 Optional,
76 Strict,
77 Many,
78}
79
80pub struct Processor<W: fmt::Write> {
81 out: W,
82 kind_names_map: HashMap<String, String>,
83 slice: u32,
84 is_token_decl: bool,
85 exports: HashMap<String, String>,
86 decl_name: String,
87 refs_bound: HashMap<String, UsedBound>,
88 is_tokens: HashSet<String>,
89 methods: HashMap<String, Vec<(String, Method)>>,
90}
91
92impl<W: fmt::Write> From<W> for Processor<W> {
93 fn from(out: W) -> Self {
94 Self {
95 out,
96 kind_names_map: HashMap::new(),
97 slice: 0,
98 is_token_decl: false,
99 exports: HashMap::new(),
100 decl_name: String::new(),
101 refs_bound: HashMap::new(),
102 is_tokens: HashSet::new(),
103 methods: HashMap::new(),
104 }
105 }
106}
107
108const PRE_DEFINE_ITEMS: &str = {
109r#"// Generated by rowan-peg, do not edit it
110use rowan::{ast::{support, AstChildren, AstNode}, Language};
111
112macro_rules! classes {
113 ($($pat:tt)*) => {
114 |s, i| ::char_classes::FirstElem::first_elem(&s[i..])
115 .filter(::char_classes::any!($($pat)*))
116 .map_or(::peg::RuleResult::Failed, |ch| {
117 ::peg::RuleResult::Matched(i+ch.len_utf8(), ())
118 })
119 };
120}
121macro_rules! decl_ast_node {
122 ($node:ident, $kind:ident) => {
123 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
124 pub struct $node(SyntaxNode);
125 impl AstNode for $node {
126 type Language = Lang;
127
128 fn syntax(&self) -> &rowan::SyntaxNode<Self::Language> {
129 &self.0
130 }
131
132 fn can_cast(kind: <Self::Language as Language>::Kind) -> bool {
133 kind == SyntaxKind::$kind
134 }
135
136 fn cast(node: rowan::SyntaxNode<Self::Language>) -> Option<Self> {
137 if Self::can_cast(node.kind()) {
138 Some(Self(node))
139 } else {
140 None
141 }
142 }
143 }
144 impl core::fmt::Display for $node {
145 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
146 core::fmt::Display::fmt(self.syntax(), f)
147 }
148 }
149 };
150}
151
152#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
153pub enum Lang {}
154impl Language for Lang {
155 type Kind = SyntaxKind;
156
157 fn kind_to_raw(kind: Self::Kind) -> ::rowan::SyntaxKind {
158 kind.into()
159 }
160
161 fn kind_from_raw(raw: ::rowan::SyntaxKind) -> Self::Kind {
162 raw.into()
163 }
164}
165
166pub type SyntaxNode = ::rowan::SyntaxNode<Lang>;
167pub type SyntaxToken = ::rowan::SyntaxToken<Lang>;
168"#};
169const PRE_DEFINE_RULES: &str = r#""#;
170
171impl<W: fmt::Write> Processor<W> {
172 fn gen_tok_wrap<F, R>(&mut self, kind: &str, f: F) -> R
173 where F: FnOnce(&mut Self) -> R,
174 {
175 write!(self.out, "(g:({{state.quiet().guard_token({kind})}}) s:$((").unwrap();
176 let result = f(self);
177 write!(self.out, ")) {{g.accept_token(s)}})").unwrap();
178 result
179 }
180
181 fn gen_node_wrap<F, R>(&mut self, kind: &str, f: F) -> R
182 where F: FnOnce(&mut Self) -> R,
183 {
184 write!(self.out, "(g:({{state.guard({kind})}}) (").unwrap();
185 let result = f(self);
186 write!(self.out, ") {{g.accept()}})").unwrap();
187 result
188 }
189
190 fn gen_quiet_wrap<F, R>(&mut self, f: F) -> R
191 where F: FnOnce(&mut Self) -> R,
192 {
193 write!(self.out, "(g:({{state.quiet().guard_none()}}) (quiet!{{").unwrap();
194 let result = f(self);
195 write!(self.out, "}}) {{g.accept_none()}})").unwrap();
196 result
197 }
198
199 fn gen_back_wrap<F, R>(&mut self, f: F) -> R
200 where F: FnOnce(&mut Self) -> R,
201 {
202 write!(self.out, "(g:({{state.guard_none()}})(").unwrap();
203 let result = f(self);
204 write!(self.out, "){{g.accept_none()}})").unwrap();
205 result
206 }
207
208 fn regist_name(&mut self, name: &str) -> (String, String) {
209 let name = utils::rule_name_of(name);
210 let kind_name = self.kind_names_map.entry(name.to_owned())
211 .or_insert_with(|| utils::kind_name_of(self.exports.get(&name).unwrap_or(&name)));
212 (name, kind_name.clone())
213 }
214
215 fn regist_tok_name(&mut self, token: &SyntaxToken) -> Result<(String, String)> {
216 let content = if token.kind() == SyntaxKind::STRING { value::string(token) } else { value::matches(token) };
217 if content.is_empty() {
218 return Err(Error::EmptyLiteral(token.clone()));
219 }
220 let (name, kind_name) = if let Some(name) = utils::punct_name_of(content) {
221 (name.to_owned(), utils::kind_name_of(&name))
222 } else if is_xid_start(content.chars().next().unwrap())
223 && content.chars().all(|ch| matches!(ch, '-' | '_') || is_xid_continue(ch))
224 {
225 let name = utils::rule_name_of(&format!("{content}_kw"));
226 let kind_name = utils::kind_name_of(&name);
227 (name, kind_name)
228 } else {
229 return Err(Error::UnknownLiteral(token.to_owned()));
230 };
231
232 self.is_tokens.insert(name.clone());
233 self.kind_names_map.insert(name.clone(), kind_name.clone());
234
235 Ok((name, kind_name))
236 }
237
238 fn add_bound(&mut self, name: impl Into<String>) {
239 if !self.is_token_decl {
240 let mut name = name.into();
241 if let Some(renamed_name) = self.exports.get(&name) {
242 name = renamed_name.to_owned();
243 }
244 *self.refs_bound.entry(name).or_default() += 1;
245 }
246 }
247
248 fn dis_refs_bound<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
249 let refs_bound = self.take_refs_bound();
250 let result = f(self);
251 self.refs_bound = refs_bound;
252 result
253 }
254
255 #[must_use]
256 fn take_refs_bound(&mut self) -> HashMap<String, UsedBound> {
257 take(&mut self.refs_bound)
258 }
259
260 pub fn start_process(&mut self, decl_list: &DeclList) -> Result<()> {
261 for export in decl_list.export_list().iter().flat_map(|list| list.exports()) {
262 let name = export.ident();
263 let new_name = export.named()
264 .map_or(name.clone(), |it| it.ident());
265 self.exports.insert(
266 utils::rule_name_of(name.text()),
267 utils::rule_name_of(new_name.text()),
268 );
269 }
270
271 writeln!(self.out, "{PRE_DEFINE_ITEMS}").unwrap();
272 writeln!(self.out, "::peg::parser!(pub grammar parser<'b>(state: \
273 &'b ::rowan_peg_utils::ParseState<'input>) for str {{").unwrap();
274 writeln!(self.out, " use SyntaxKind::*;").unwrap();
275 writeln!(self.out, "{PRE_DEFINE_RULES}").unwrap();
276 for decl in decl_list.decls() {
277 self.process_decl(decl)?;
278 }
279 writeln!(self.out, "}});").unwrap();
280 writeln!(self.out, "#[repr(u16)]").unwrap();
281 writeln!(self.out, "#[allow(non_camel_case_types)]").unwrap();
282 writeln!(self.out, "#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]").unwrap();
283 writeln!(self.out, "pub enum SyntaxKind {{").unwrap();
284 let mut first = true;
285 let mut last = None;
286 for kind in self.kind_names_map.values() {
287 first.to_false(|| {
288 writeln!(self.out, " {kind} = 0,").unwrap();
289 }).unwrap_or_else(|| {
290 writeln!(self.out, " {kind},").unwrap();
291 });
292 last = kind.into();
293 }
294 writeln!(self.out, "}}").unwrap();
295 writeln!(self.out, "impl From<::rowan::SyntaxKind> for SyntaxKind {{ \
296 fn from(kind: ::rowan::SyntaxKind) -> Self {{ \
297 ::core::assert!(kind.0 <= Self::{} as u16); \
298 unsafe {{ ::core::mem::transmute::<u16, SyntaxKind>(kind.0) }} \
299 }} \
300 }}", last.unwrap()).unwrap();
301 writeln!(self.out, "impl From<SyntaxKind> for ::rowan::SyntaxKind {{ \
302 fn from(kind: SyntaxKind) -> Self {{ \
303 ::rowan::SyntaxKind(kind as u16) \
304 }} \
305 }}").unwrap();
306 for (rule_name, mut methods) in self.methods.drain() {
307 if self.is_tokens.contains(&rule_name) { continue }
308 let node_name = utils::node_name_of(&rule_name);
309 let node_kind = utils::kind_name_of(&rule_name);
310 methods.sort_by(|a, b| a.0.cmp(&b.0));
311
312 writeln!(self.out, "decl_ast_node!({node_name}, {node_kind});").unwrap();
313 writeln!(self.out, "impl {node_name} {{").unwrap();
314 for (child_name, method) in methods {
315 let is_token = self.is_tokens.contains(&child_name);
316 let mut base_ty = if is_token {
317 "SyntaxToken".to_owned()
318 } else {
319 utils::node_name_of(&child_name)
320 };
321 match method {
322 Method::Optional => base_ty = format!("Option<{base_ty}>"),
323 Method::Strict => (),
324 Method::Many => base_ty = format!("AstChildren<{base_ty}>"),
325 }
326 let body = if is_token {
327 let kind = utils::kind_name_of(&child_name);
328 match method {
329 Method::Optional => format!("support::token(self.syntax(), SyntaxKind::{kind})"),
330 Method::Strict => format!("support::token(self.syntax(), SyntaxKind::{kind}).unwrap()"),
331 Method::Many => continue,
332 }
333 } else {
334 match method {
335 Method::Optional => "support::child(self.syntax())",
336 Method::Strict => "support::child(self.syntax()).unwrap()",
337 Method::Many => "support::children(self.syntax())",
338 }.into()
339 };
340 let method_name = if method == Method::Many {
341 if child_name.ends_with('s') {
342 format!("{child_name}es")
343 } else {
344 format!("{child_name}s")
345 }
346 } else {
347 child_name
348 };
349 writeln!(self.out, " pub fn {method_name}(&self) -> {base_ty} {{").unwrap();
350 writeln!(self.out, " {body}").unwrap();
351 writeln!(self.out, " }}").unwrap();
352 }
353 writeln!(self.out, "}}").unwrap();
354 }
355 Ok(())
356 }
357
358 fn decl_is_token(&self, decl: &Decl) -> bool {
359 let Some(list) = utils::one_elem(decl.pat_choice().pat_lists()) else { return false };
360 let Some(op) = utils::one_elem(list.pat_ops()) else { return false };
361 if op.dollar().is_some() {
362 return true;
363 }
364 op.pat_atom().syntax().text_range() != op.syntax().text_range()
365 && op.pat_atom().string().is_some()
366 }
367
368 fn process_decl(&mut self, decl: Decl) -> Result<()> {
369 let (name, kind_name) = self.regist_name(decl.named().ident().text());
370 self.is_token_decl = self.decl_is_token(&decl);
371 self.decl_name = name;
372 let name = &self.decl_name;
373 let mut vis = "";
374
375 if self.is_token_decl {
376 self.is_tokens.insert(name.clone());
377 }
378 if let Some(new_name) = self.exports.get(name) {
379 if name == new_name {
380 vis = "pub ";
381 } else {
382 writeln!(self.out, " pub rule {new_name}() = {name}").unwrap();
383 }
384 }
385
386 self.refs_bound.clear();
387 write!(self.out, " {vis}rule {name}() = ").unwrap();
388 if self.is_token_decl {
389 write!(self.out, "()").unwrap();
390 self.process_pat_choice(decl.pat_choice())?;
391 } else {
392 self.gen_node_wrap(&kind_name, |this| {
393 this.process_pat_choice(decl.pat_choice())
394 })?;
395 }
396
397 writeln!(self.out).unwrap();
398 let methods = self.refs_bound.iter().filter_map(|(name, bound)| {
399 let ty = match bound {
400 UsedBound(0, 0) => return None,
401 UsedBound(0, 1) => Method::Optional,
402 UsedBound(1, 1) => Method::Strict,
403 _ => Method::Many,
404 };
405 Some((name.clone(), ty))
406 }).collect();
407 let name = self.exports.get(&self.decl_name).unwrap_or(&self.decl_name);
408 self.methods.insert(name.clone(), methods);
409 Ok(())
410 }
411
412 fn process_pat_choice(&mut self, patchoice: PatChoice) -> Result<()> {
413 let mut first = true;
414 let refs_bound = self.take_refs_bound();
415 let mut prev_bound: Option<HashMap<String, UsedBound>> = None;
416 write!(self.out, "(").unwrap();
417 for patlist in patchoice.pat_lists() {
418 first.in_false(|| write!(self.out, " / ").unwrap());
419 write!(self.out, "()").unwrap();
420 self.gen_back_wrap(|this| this.process_pat_list(patlist))?;
421 if let Some(prev_bound) = &mut prev_bound {
422 self.merge_cover_to(prev_bound);
423 } else {
424 prev_bound = Some(self.take_refs_bound());
425 }
426 }
427 assert_eq!(self.refs_bound.len(), 0);
428 self.merge_add(refs_bound);
429 self.merge_add(prev_bound.unwrap());
430 if let Some(expected) = patchoice.pat_expect() {
431 let name = value::label(&expected.label());
432 write!(self.out, " / expected!({name:?})").unwrap();
433 }
434 write!(self.out, ")").unwrap();
435 Ok(())
436 }
437
438 fn merge_cover_to(&mut self, prev_bound: &mut HashMap<String, UsedBound>) {
439 for key in self.refs_bound.keys() {
440 if !prev_bound.contains_key(key) {
441 prev_bound.insert(key.clone(), UsedBound::default());
442 }
443 }
444 for (key, value) in &mut *prev_bound {
445 let other = self.refs_bound.get(key).copied().unwrap_or_default();
446 *value = value.cover(other);
447 }
448 self.refs_bound.clear();
449 }
450
451 fn merge_add(&mut self, refs_bound: HashMap<String, UsedBound>) {
452 for (key, value) in refs_bound {
453 *self.refs_bound.entry(key).or_default() += value;
454 }
455 }
456
457 fn process_pat_list(&mut self, patlist: PatList) -> Result<()> {
458 let mut first = true;
459 for patop in patlist.pat_ops() {
460 first.in_false(|| write!(self.out, " ").unwrap());
461 self.process_patop(patop)?;
462 }
463 Ok(())
464 }
465
466 fn process_patop(&mut self, patop: PatOp) -> Result<()> {
467 let atom = patop.pat_atom();
468 if patop.amp().is_some() {
469 write!(self.out, "&").unwrap();
470 self.gen_quiet_wrap(|this| this.dis_refs_bound(|this| this.process_patatom(atom)))?;
471 } else if patop.bang().is_some() {
472 write!(self.out, "!").unwrap();
473 self.gen_quiet_wrap(|this| this.dis_refs_bound(|this| this.process_patatom(atom)))?;
474 } else if patop.tilde().is_some() {
475 write!(self.out, "quiet!{{").unwrap();
476 self.dis_refs_bound(|this| this.process_patatom(atom))?;
477 write!(self.out, "}}").unwrap();
478 } else if patop.dollar().is_some() {
479 if self.is_token_decl && self.slice == 0 {
480 let name = &self.decl_name.clone();
481 let (_, kind_name) = self.regist_name(name);
482 self.slice += 1;
483 self.gen_tok_wrap(&kind_name, |this| {
484 this.dis_refs_bound(|this| this.process_patatom(atom))
485 })?;
486 self.slice -= 1;
487 } else {
488 return Err(Error::DisallowedSlice(patop.syntax().clone()));
489 }
490 } else if let Some(repeat) = patop.repeat() {
491 let refs_bound = self.take_refs_bound();
492 self.gen_back_wrap(|this| this.process_patatom(atom))?;
493 let (lower_bound, upper_bound) = repeat.count_bounds();
494 match (lower_bound, upper_bound) {
495 (1, None) => write!(self.out, "+"),
496 (0, None) => write!(self.out, "*"),
497 (lower, None) => write!(self.out, "*<{lower},>"),
498 (lower, Some(upper)) => write!(self.out, "*<{lower},{upper}>"),
499 }.unwrap();
500 let repeat_meta = UsedBound(
501 lower_bound.try_into().unwrap(),
502 upper_bound.unwrap_or(255).try_into().unwrap(),
503 );
504 self.refs_bound.iter_mut().for_each(|(_, bound)| *bound *= repeat_meta);
505 self.merge_add(refs_bound);
506 } else {
507 self.process_patatom(atom)?;
508 }
509 Ok(())
510 }
511
512 fn process_patatom(&mut self, atom: PatAtom) -> Result<()> {
513 if atom.l_paren().is_some() {
514 self.process_pat_choice(atom.pat_choice().unwrap())?;
515 } else if atom.l_brack().is_some() {
516 let refs_bound = self.take_refs_bound();
517 self.gen_back_wrap(|this| this.process_pat_choice(atom.pat_choice().unwrap()))?;
518 write!(self.out, "?").unwrap();
519 self.refs_bound.iter_mut().for_each(|(_, bound)| bound.0 = 0);
520 self.merge_add(refs_bound);
521 } else if let Some(ident) = atom.ident() {
522 let name = utils::rule_name_of(ident.text());
523 write!(self.out, "{}()", name).unwrap();
524 self.add_bound(name);
525 } else if let Some(string) = atom.string() {
526 self.tok_or_in_slice(&string)?;
527 } else if let Some(matches) = atom.matches()
528 && value::matches(&matches).chars().count() == 1
529 {
530 self.tok_or_in_slice(&matches)?;
532 } else if let Some(matches) = atom.matches() {
533 if self.slice == 0 {
534 return Err(Error::MatchesWithoutSlice(matches));
535 }
536 let content = value::matches(&matches);
537 if content.starts_with('^') {
538 write!(self.out, "#{{classes!(^\"{}\")}}", &content[1..]).unwrap();
539 } else {
540 write!(self.out, "#{{classes!(\"{content}\")}}").unwrap();
541 }
542 } else {
543 unreachable!()
544 }
545 Ok(())
546 }
547
548 fn tok_or_in_slice(&mut self, token: &SyntaxToken) -> Result<()> {
549 let content = if token.kind() == SyntaxKind::STRING {
550 value::string(token)
551 } else {
552 value::matches(token)
553 };
554 if self.slice == 0 {
555 let (name, kind_name) = self.regist_tok_name(&token)?;
556 self.add_bound(name);
557 self.gen_tok_wrap(&kind_name, |this| {
558 write!(this.out, "{content:?}").unwrap();
559 });
560 } else {
561 write!(self.out, "{content:?}").unwrap();
562 }
563 Ok(())
564 }
565}
566
567#[cfg(test)]
568mod tests {
569 use rowan::TextSize;
570
571 use super::*;
572
573 #[test]
574 fn full_parser() {
575 let s = r#"
576;; use ABNF like grammar
577;; char-val to case-sensitive
578;; prose-val -> regexp
579;; add peg lookaheads `!` `&`
580;; add quiet `~`
581;; add slice `$`
582;; remove num-var
583;;
584;; vim:nowrap
585
586comment = ~<;[^\n]*(?:\n|$)> @comment
587_ = ~<[ \t\r\n]*> [comment _]
588ident = ~<(?![0-9])(?:[0-9a-zA-Z\-_]|[^\x00-\xa0])+> @ident
589number = ~<[0-9]+> @number
590string = ~(<"> <[^\"\r\n]*> <">) @string
591match = ~("<" <[^\x3e\r\n]*> ">") @match
592label = ident / string
593repeat = "+"
594 / "*" [number]
595 / number ["*" [number]]
596patatom = ident !(_ "=") ; a rule reference
597 / string ; keyword
598 / match ; regular expressions
599 / "[" _ patchoice _ "]" ; optional
600 / "(" _ patchoice _ ")" ; simple paren
601 / "{" _ patchoice _ "}" ; list group brace
602patrepeat = repeat _ patatom
603 / patatom
604patop = "&" patrepeat ; positive lookahead
605 / "!" patrepeat ; negative lookahead
606 / "~" patrepeat ; quiet
607 / "$" patrepeat ; slice
608 / patrepeat
609patlist = patop *(_ patop)
610patchoice = patlist *(_ "/" _ patlist)
611 *(_ "@" label); extra expected branch
612decl = ident _ "=" _ patchoice
613decl-list = +(_ decl) _
614 "#;
615 let mut state = rowan_peg_utils::ParseState::default();
616 parser::decl_list(s, &state).unwrap();
617 dbg!(&state);
618 let node = SyntaxNode::new_root(state.finish());
619 dbg!(&node);
620 assert_eq!(TextSize::of(s), node.text_range().end());
621 dbg!(&s.len());
622 let decl_list = DeclList::cast(node).unwrap();
623 println!("{decl_list}")
624 }
625}