1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

use crate::{dump, Module};

use pyo3::prelude::*;

/// Takes a string of bytes and returns the Python-tokenized version of it.
/// use python_ast::parse;
///
/// ```Rust
/// fn read_python_file(input: std::path::Path) {
///    let py = read_to_string(input).unwrap();
///    let ast = parse(&py, "__main__").unwrap();
///
///    println!("{:?}", ast);
///}
/// ```
pub fn parse<S1: Into<String>, S2: Into<String>>(input: S1, filename: S2) -> PyResult<Module> {

    let pymodule_code = include_str!("__init__.py");

    Python::with_gil(|py| -> PyResult<Module> {
        // We want to call tokenize.tokenize from Python.
        let pymodule = PyModule::from_code(py, pymodule_code, "parser.py", "parser")?;
        let t = pymodule.getattr("parse")?;
        assert!(t.is_callable());
        let args = (input.into(), filename.into());

        let py_tree = t.call1(args)?;
        log::debug!("py_tree: {}", dump(py_tree, Some(4))?);

        let tree: Module = py_tree.extract()?;

        Ok(tree)
    })
}