ripex 0.2.0

Multi-language structural parsing and fact extraction.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

pub mod declarations;
pub mod expressions;
pub mod jsx;
pub mod modules;
pub mod patterns;
pub mod recovery;
pub mod state;
pub mod statements;
pub mod typescript;

pub use state::{Context, Parser};

use crate::arena::Arena;
use crate::diagnostics::{DiagnosticCode, ParseError};
use crate::js::ast::{Expr, Module, Program, Script, Stmt};
use crate::js::config::ParserOptions;
use crate::js::lexer::{Lexer, Token};

pub fn parse_program(
    source: &str,
    options: &ParserOptions,
) -> (Program, Vec<ParseError>, Arena<Expr>) {
    let (program, errors, arena, _) = parse_program_with_comments(source, options);
    (program, errors, arena)
}

/// Parse a program and retain comments for source-preserving tooling.
pub fn parse_program_with_comments(
    source: &str,
    options: &ParserOptions,
) -> (
    Program,
    Vec<ParseError>,
    Arena<Expr>,
    Vec<crate::js::lexer::Comment>,
) {
    if source.len() > crate::limits::MAX_INPUT_SIZE {
        let err = ParseError::new(DiagnosticCode::InputTooLarge, crate::span::Span::ZERO);
        let ast = Arena::new();
        return (
            Program::Script(Script {
                span: crate::span::Span::ZERO,
                body: Vec::new(),
            }),
            vec![err],
            ast,
            Vec::new(),
        );
    }
    let (tokens, token_limit_exceeded) = tokenize(source, options);
    let mut parser = Parser::new(tokens, options);
    let program = parser.parse_program();
    if token_limit_exceeded {
        parser.errors.push(ParseError::new(
            DiagnosticCode::TokenLimitExceeded,
            parser.current_token().span,
        ));
    }
    let errors = parser.errors.clone();
    let trailing_comments = if options.capture_comments && parser.is_eof() {
        parser
            .tokens
            .get(parser.pos)
            .map(|token| token.leading_comments.clone())
            .unwrap_or_default()
    } else {
        Vec::new()
    };
    let ast = parser.ast;
    let mut comments = parser.comments;
    comments.extend(trailing_comments);
    (program, errors, ast, comments)
}

pub fn parse_module(
    source: &str,
    options: &ParserOptions,
) -> (Module, Vec<ParseError>, Arena<Expr>) {
    let mut opts = options.clone();
    opts.source_type = crate::js::config::SourceType::Module;
    if source.len() > crate::limits::MAX_INPUT_SIZE {
        return (
            Module {
                span: crate::span::Span::ZERO,
                body: Vec::new(),
            },
            vec![ParseError::new(
                DiagnosticCode::InputTooLarge,
                crate::span::Span::ZERO,
            )],
            Arena::new(),
        );
    }
    let (tokens, token_limit_exceeded) = tokenize(source, &opts);
    let mut parser = Parser::new(tokens, &opts);
    let module = parser.parse_module();
    if token_limit_exceeded {
        parser.errors.push(ParseError::new(
            DiagnosticCode::TokenLimitExceeded,
            parser.current_token().span,
        ));
    }
    let errors = parser.errors.clone();
    let ast = parser.ast;
    (module, errors, ast)
}

pub fn parse_script(
    source: &str,
    options: &ParserOptions,
) -> (Vec<Stmt>, Vec<ParseError>, Arena<Expr>) {
    if source.len() > crate::limits::MAX_INPUT_SIZE {
        return (
            Vec::new(),
            vec![ParseError::new(
                DiagnosticCode::InputTooLarge,
                crate::span::Span::ZERO,
            )],
            Arena::new(),
        );
    }
    let (tokens, token_limit_exceeded) = tokenize(source, options);
    let mut parser = Parser::new(tokens, options);
    let stmts = parser.parse_script();
    if token_limit_exceeded {
        parser.errors.push(ParseError::new(
            DiagnosticCode::TokenLimitExceeded,
            parser.current_token().span,
        ));
    }
    let errors = parser.errors.clone();
    let ast = parser.ast;
    (stmts, errors, ast)
}

fn tokenize(source: &str, options: &ParserOptions) -> (Vec<Token>, bool) {
    let mut lexer = Lexer::with_jsx(source, options.features.jsx);
    let tokens = lexer.by_ref().collect();
    (tokens, lexer.token_limit_exceeded())
}