squawk-parser 2.66.0

// via https://github.com/rust-lang/rust-analyzer/blob/d8887c0758bbd2d5f752d5bd405d4491e90e7ed6/crates/parser/src/shortcuts.rs
//
// Permission is hereby granted, free of charge, to any
// person obtaining a copy of this software and associated
// documentation files (the "Software"), to deal in the
// Software without restriction, including without
// limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software
// is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice
// shall be included in all copies or substantial portions
// of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

use std::mem;

use crate::{
    lexed_str::LexedStr,
    output::{Output, Step},
    syntax_kind::SyntaxKind,
};

#[derive(Debug)]
pub enum StrStep<'a> {
    Token { kind: SyntaxKind, text: &'a str },
    Enter { kind: SyntaxKind },
    Exit,
    Error { msg: &'a str, pos: usize },
}

enum State {
    PendingEnter,
    Normal,
    PendingExit,
}

struct Builder<'a, 'b> {
    lexed: &'a LexedStr<'a>,
    pos: usize,
    state: State,
    sink: &'b mut dyn FnMut(StrStep<'_>),
}

impl Builder<'_, '_> {
    fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
        match mem::replace(&mut self.state, State::Normal) {
            State::PendingEnter => unreachable!(),
            State::PendingExit => (self.sink)(StrStep::Exit),
            State::Normal => (),
        }
        self.eat_trivias();
        self.do_token(kind, n_tokens as usize);
    }

    fn enter(&mut self, kind: SyntaxKind) {
        match mem::replace(&mut self.state, State::Normal) {
            State::PendingEnter => {
                (self.sink)(StrStep::Enter { kind });
                // No need to attach trivias to previous node: there is no
                // previous node.
                return;
            }
            State::PendingExit => (self.sink)(StrStep::Exit),
            State::Normal => (),
        }

        self.eat_trivias();
        (self.sink)(StrStep::Enter { kind });
    }

    fn exit(&mut self) {
        match mem::replace(&mut self.state, State::PendingExit) {
            State::PendingEnter => unreachable!(),
            State::PendingExit => (self.sink)(StrStep::Exit),
            State::Normal => (),
        }
    }

    fn eat_trivias(&mut self) {
        while self.pos < self.lexed.len() {
            let kind = self.lexed.kind(self.pos);
            if !kind.is_trivia() {
                break;
            }
            self.do_token(kind, 1);
        }
    }

    fn do_token(&mut self, kind: SyntaxKind, n_tokens: usize) {
        let text = &self.lexed.range_text(self.pos..self.pos + n_tokens);
        self.pos += n_tokens;
        (self.sink)(StrStep::Token { kind, text });
    }
}

impl LexedStr<'_> {
    pub fn to_input(&self) -> crate::Input {
        let mut res = crate::Input::default();
        let mut was_joint = false;
        for i in 0..self.len() {
            let kind = self.kind(i);
            if kind.is_trivia() {
                was_joint = false;
                // skip over any triva since the parser shouldn't have to deal
                // with it
            } else {
                if was_joint {
                    res.was_joint();
                }
                if kind == SyntaxKind::IDENT {
                    let contextual_kind = SyntaxKind::from_contextual_keyword(self.text(i))
                        .unwrap_or(SyntaxKind::IDENT);
                    res.push_ident(contextual_kind);
                } else {
                    res.push(kind);
                }
                was_joint = true;
            }
        }
        res
    }

    /// NB: only valid to call with Output from Reparser/TopLevelEntry.
    pub fn intersperse_trivia(&self, output: &Output, sink: &mut dyn FnMut(StrStep<'_>)) -> bool {
        let mut builder = Builder {
            lexed: self,
            pos: 0,
            state: State::PendingEnter,
            sink,
        };

        for event in output.iter() {
            match event {
                Step::Token {
                    kind,
                    n_input_tokens: n_raw_tokens,
                } => builder.token(kind, n_raw_tokens),
                Step::Enter { kind } => builder.enter(kind),
                Step::Exit => builder.exit(),
                Step::Error { msg } => {
                    let text_pos = builder.lexed.text_start(builder.pos);
                    (builder.sink)(StrStep::Error { msg, pos: text_pos });
                }
            }
        }

        match mem::replace(&mut builder.state, State::Normal) {
            State::PendingExit => {
                builder.eat_trivias();
                (builder.sink)(StrStep::Exit);
            }
            State::PendingEnter | State::Normal => unreachable!(),
        }

        // is_eof?
        builder.pos == builder.lexed.len()
    }
}

#[cfg(test)]
mod tests {
    use super::LexedStr;
    use crate::SyntaxKind;

    fn kinds(text: &str) -> Vec<(SyntaxKind, SyntaxKind)> {
        let lexed = LexedStr::new(text);
        let input = lexed.to_input();
        (0..text.split_whitespace().count())
            .map(|i| (input.kind(i), input.contextual_kind(i)))
            .collect()
    }

    #[test]
    fn plpgsql_keywords_stay_idents_with_a_contextual_kind() {
        assert_eq!(
            kinds("message raise elsif"),
            vec![
                (SyntaxKind::IDENT, SyntaxKind::MESSAGE_KW),
                (SyntaxKind::IDENT, SyntaxKind::RAISE_KW),
                (SyntaxKind::IDENT, SyntaxKind::ELSIF_KW),
            ]
        );
    }

    #[test]
    fn sql_keywords_are_not_contextual() {
        assert_eq!(
            kinds("select begin declare"),
            vec![
                (SyntaxKind::SELECT_KW, SyntaxKind::EOF),
                (SyntaxKind::BEGIN_KW, SyntaxKind::EOF),
                (SyntaxKind::DECLARE_KW, SyntaxKind::EOF),
            ]
        );
    }

    #[test]
    fn plain_idents_have_no_contextual_kind() {
        assert_eq!(
            kinds("foo bar"),
            vec![
                (SyntaxKind::IDENT, SyntaxKind::IDENT),
                (SyntaxKind::IDENT, SyntaxKind::IDENT),
            ]
        );
    }
}