python-syntax 0.1.0

// vi:syntax=rust

use std::{
    default::Default,
    iter,
};

use crate::{
    ast,
    error::Error,
    source::{Location, Span},
    token::Token,
};

grammar;

pub Program: ast::Program = {
    MODULE <Module> => ast::Program::Module(<>),
    INTERACTIVE <Interactive> => ast::Program::Interactive(<>),
    EVAL <Eval> => ast::Program::Eval(<>),
};

Module: ast::Module = {
    NEWLINE* <(<Statement> NEWLINE*)*> EOF => ast::Module {
        body: <>.into_iter().flatten().collect(),
    },
};

Interactive: ast::Interactive = {
    NEWLINE => ast::Interactive { body: vec![] },
    <body:SimpleStatement> => ast::Interactive {<>},
    <s:CompoundStatement> NEWLINE => ast::Interactive { body: vec![<>] },
};

Eval: ast::Eval = {
    <TestList> NEWLINE* EOF => ast::Eval { body: Box::new(<>) },
};

Decorator: ast::Expression = {
    "@" <DecoratorName> NEWLINE,
    "@" <l:@L> <d:DecoratorName> "(" <a:ArgList?> ")" <r:@R> NEWLINE => {
        let (args, keywords) = a.unwrap_or_default();
        ast::Expression::Call(ast::Call {
        func: Box::new(d),
            span: Span::new(l, r),
            args,
            keywords,
        })
    },
};

DecoratorName: ast::Expression = {
    <l:@L> <id:NAME> <r:@R> => ast::Expression::Name(ast::Name {
        span: Span::new(l, r),
        id,
    }),
    <l:@L> <d:DecoratorName> "." <n:NAME> <r:@R> => {
        ast::Expression::Attribute(ast::Attribute {
            span: Span::new(l, r),
            value: Box::new(d),
            attr: n,
        })
    },
};

Decoratable: ast::Statement = {ClassDef, FuncDef, AsyncFuncDef};

Decorated: ast::Statement = {
    <l:@L> <v:Decorator+> <s:Decoratable> => {
        let mut s = s;
        match s {
            ast::Statement::ClassDef(ast::ClassDef {
                ref mut span, ref mut decorator_list, .. })
            | ast::Statement::FunctionDef(ast::FunctionDef {
                ref mut span, ref mut decorator_list, .. })
            | ast::Statement::AsyncFunctionDef(ast::AsyncFunctionDef {
                ref mut span, ref mut decorator_list, .. })
            => {
                span.start = l;
                *decorator_list = v;
            }
            _ => unreachable!(),
        }
        s
    },
};

AsyncFuncDef: ast::Statement = {
    <l:@L> "async" <s:FuncDef> <r:@R> => {
        match s {
            ast::Statement::FunctionDef(ast::FunctionDef {
                name,
                args,
                body,
                decorator_list,
                returns,
                ..
            }) => ast::Statement::AsyncFunctionDef(ast::AsyncFunctionDef {
                span: Span::new(l, r),
                name,
                args,
                body,
                decorator_list,
                returns,
            }),
            _ => unreachable!(),
        }
    },
};

FuncDef: ast::Statement = {
    <l:@L> "def" <n:NAME> "(" <v:TypedParamList?> ")" <t:("->" <Test>)?> ":" <s:Suite> <r:@R>
    => {
        ast::Statement::FunctionDef(ast::FunctionDef {
            span: Span::new(l, r),
            name: n,
            args: v.map(Box::new).unwrap_or_default(),
            body: s,
            decorator_list: vec![],
            returns: t.map(Box::new),
        })
    },
};

TypedParamList = ParamList<TypedParam>;
UntypedParamList = ParamList<UntypedParam>;

ParamList<P>: ast::Arguments = {
    <v:Comma<SingleParam<P>>> <s:("," <StarParamList<P>>)?> ","? => {
        let mut s = s.unwrap_or_default();
        s.args = v;
        s
    },
    <StarParamList<P>> ","?,
};

#[inline]
StarParamList<P>: ast::Arguments = {
    <va:StarParam<P>> <v:("," <Comma<SingleParam<P>>>)?> <kw:("," <KwParam<P>>)?> => {
        ast::Arguments {
            vararg: Some(va),
            kwonlyargs: v.unwrap_or_default(),
            kwarg: kw,
            .. Default::default()
        }
    },
    "*" "," <v:Comma<SingleParam<P>>> <kw:("," <KwParam<P>>)?> => {
        ast::Arguments {
            kwonlyargs: v,
            kwarg: kw,
            .. Default::default()
        }
    },
    KwParam<P> => ast::Arguments {
        kwarg: Some(<>),
        .. Default::default()
    },
};

SingleParam<P>: ast::Arg = {OptionalParam<P>, RequiredParam<P>};

RequiredParam<P>: ast::Arg = <l:@L> <p:P> <r:@R> => {
    ast::Arg {
        span: Span::new(l, r),
        arg: p.0,
        annotation: p.1.map(Box::new),
        kind: ast::ArgKind::Required,
    }
};

OptionalParam<P>: ast::Arg = <l:@L> <p:P> <t:("=" <Test>)> <r:@R> => {
    ast::Arg {
        span: Span::new(l, r),
        arg: p.0,
        annotation: p.1.map(Box::new),
        kind: ast::ArgKind::Optional(Box::new(t)),
    }
};

StarParam<P>: ast::Arg = <l:@L> "*" <p:P> <r:@R> => {
    ast::Arg {
        span: Span::new(l, r),
        arg: p.0,
        annotation: p.1.map(Box::new),
        kind: ast::ArgKind::Vararg,
    }
};

KwParam<P>: ast::Arg = <l:@L> "**" <p:P> <r:@R> => {
    ast::Arg {
        span: Span::new(l, r),
        arg: p.0,
        annotation: p.1.map(Box::new),
        kind: ast::ArgKind::Kwarg,
    }
};
 
TypedParam: (String, Option<ast::Expression>) = <n:NAME> <t:(":" <Test>)?> => (n, t);
UntypedParam: (String, Option<ast::Expression>) = <NAME> => (<>, None);

Statement: Vec<ast::Statement> = {
    SimpleStatement,
    <CompoundStatement> => vec![<>],
};

SimpleStatement: Vec<ast::Statement> = <Sep<SmallStatement, ";">> ";"? NEWLINE;

SmallStatement: ast::Statement = {
    ExprStatement,
    DelStatement,
    PassStatement,
    FlowStatement,
    ImportStatement,
    GlobalStatement,
    NonlocalStatement,
    AssertStatement,
};

ExprStatement: ast::Statement = {
    <l:@L> <t:TestListStarExpr> <r:@R> => {
        ast::Statement::Expr(ast::Expr {
            span: Span::new(l, r),
            value: Box::new(t),
        })
    },
    <l:@L> <t:TestListStarExpr> ":" <a:Test> <v:("=" <Test>)?> <r:@R> => {
        let simple = match t {
            ast::Expression::Name(ast::Name {
                span: Span { start, .. },
                ..
            }) if start == l => true,
            _ => false,
        };
        ast::Statement::AnnAssign(ast::AnnAssign {
            span: Span::new(l, r),
            target: Box::new(t),
            annotation: Box::new(a),
            value: v.map(Box::new),
            simple,
        })
    },
    <l:@L> <t:TestListStarExpr> <op:AugAssign> <v:AssignValue> <r:@R> => {
        ast::Statement::AugAssign(ast::AugAssign {
            span: Span::new(l, r),
            target: Box::new(t),
            op,
            value: Box::new(v),
        })
    },
    <l:@L> <t:TestListStarExpr> <v:("=" <AssignValue>)+> <r:@R> => {
        let (targets, value) = v.split_at(v.len() - 1);
        ast::Statement::Assign(ast::Assign {
            span: Span::new(l, r),
            targets: iter::once(t).chain(targets.iter().cloned()).collect(),
            value: Box::new(value[0].clone()),
        })
    },
};

AugAssign: ast::OpKind = {
    "+=" => ast::OpKind::Addition,
    "-=" => ast::OpKind::Subtraction,
    "*=" => ast::OpKind::Multiplication,
    "@=" => ast::OpKind::MatrixMultiplication,
    "/=" => ast::OpKind::Division,
    "%=" => ast::OpKind::Modulo,
    "**=" => ast::OpKind::Power,
    "<<=" => ast::OpKind::LeftShift,
    ">>=" => ast::OpKind::RightShift,
    "|=" => ast::OpKind::BitOr,
    "^=" => ast::OpKind::BitXor,
    "&=" => ast::OpKind::BitAnd,
    "//=" => ast::OpKind::FloorDivision,
};

AssignValue: ast::Expression = {YieldExpr, TestList};

DelStatement: ast::Statement = {
    <l:@L> "del" <v:Comma<ExprOrStar>> ","? <r:@R> => {
        ast::Statement::Delete(ast::Delete {
            span: Span::new(l, r),
            targets: v,
        })
    }
};

PassStatement: ast::Statement = <l:@L> "pass" <r:@R> => {
    ast::Statement::Pass(ast::Pass { span: Span::new(l, r) })
};

FlowStatement: ast::Statement = {
    BreakStatement,
    ContinueStatement,
    ReturnStatement,
    RaiseStatement,
    YieldStatement,
};

BreakStatement: ast::Statement = <l:@L> "break" <r:@R> => {
    ast::Statement::Break(ast::Break { span: Span::new(l, r) })
};

ContinueStatement: ast::Statement = <l:@L> "continue" <r:@R> => {
    ast::Statement::Continue(ast::Continue { span: Span::new(l, r) })
};

ReturnStatement: ast::Statement = <l:@L> "return" <t:TestList?> <r:@R> => {
    ast::Statement::Return(ast::Return {
        span: Span::new(l, r),
        value: t.map(Box::new),
    })
};

YieldStatement: ast::Statement = <l:@L> <e:YieldExpr> <r:@R> => {
    ast::Statement::Expr(ast::Expr {
        span: Span::new(l, r),
        value: Box::new(e),
    })
};

RaiseStatement: ast::Statement = {
    <l:@L> "raise" <e:Test?> <r:@R> => ast::Statement::Raise(ast::Raise {
        span: Span::new(l, r),
        exc: e.map(Box::new),
        cause: None,
    }),
    <l:@L> "raise" <e:Test> "from" <c:Test> <r:@R> => {
        ast::Statement::Raise(ast::Raise {
            span: Span::new(l, r),
            exc: Some(Box::new(e)),
            cause: Some(Box::new(c)),
        })
    },
};

ImportStatement: ast::Statement = {ImportName, ImportFrom};

ImportName: ast::Statement = <l:@L> "import" <v:DottedAsNames> <r:@R> => {
    ast::Statement::Import(ast::Import {
        span: Span::new(l, r),
        names: v,
    })
};

ImportFrom: ast::Statement = {
    <l:@L> "from" <m:ImportFromModule> "import" <n:ImportFromNames> <r:@R> => {
        ast::Statement::ImportFrom(ast::ImportFrom {
            span: Span::new(l, r),
            module: m.0,
            names: n,
            level: m.1,
        })
    },
};

ImportFromModule: (Option<String>, usize) = {
    <l:ImportFromLevelList?> <d:DottedName> => (Some(d), l.unwrap_or_default()),
    <ImportFromLevelList> => (None, <>),
};

ImportFromLevelList: usize = <l:ImportFromLevel+> => l.into_iter().sum();

ImportFromLevel: usize = {
    "." => 1,
    "..." => 3,
};

ImportFromNames: Vec<ast::Alias> = {
    "*" => vec![ast::Alias { identifier: "*".to_string(), asname: None }],
    "(" <ImportAsNames> ")",
    <ImportAsNames>,
};

ImportAsName: ast::Alias = <a:NAME> <b:("as" <NAME>)?> => {
    ast::Alias {
        identifier: a,
        asname: b,
    }
};

DottedAsName: ast::Alias = <d:DottedName> <n:("as" <NAME>)?> => {
    ast::Alias {
        identifier: d,
        asname: n,
    }
};

ImportAsNames: Vec<ast::Alias> = <Comma<ImportAsName>> ","?;

DottedAsNames: Vec<ast::Alias> = Comma<DottedAsName>;

DottedName: String = <v:Sep<NAME, ".">> => v.concat();

GlobalStatement: ast::Statement = {
    <l:@L> "global" <v:Comma<NAME>> <r:@R> => {
        ast::Statement::Global(ast::Global {
            span: Span::new(l, r),
            names: v,
        })
    },
};

NonlocalStatement: ast::Statement = {
    <l:@L> "nonlocal" <v:Comma<NAME>> <r:@R> => {
        ast::Statement::Nonlocal(ast::Nonlocal {
            span: Span::new(l, r),
            names: v,
        })
    },
};

AssertStatement: ast::Statement = {
    <l:@L> "assert" <t:Test> <m:("," <Test>)?> <r:@R> => {
        ast::Statement::Assert(ast::Assert {
            span: Span::new(l, r),
            test: Box::new(t),
            msg: m.map(Box::new),
        })
    },
};

CompoundStatement: ast::Statement = {
    IfStatement,
    WhileStatement,
    ForStatement,
    TryStatement,
    WithStatement,
    FuncDef,
    ClassDef,
    Decorated,
    AsyncStatement,
};

AsyncStatement: ast::Statement = {
    AsyncFuncDef,
    <l:@L> "async" <s:WithStatement> <r:@R> => {
        match s {
            ast::Statement::With(ast::With { items, body, .. }) => {
                ast::Statement::AsyncWith(ast::AsyncWith {
                    span: Span::new(l, r),
                    items,
                    body,
                })
            }
            _ => unreachable!(),
        }
    },
    <l:@L> "async" <s:ForStatement> <r:@R> => {
        match s {
            ast::Statement::For(ast::For { target, iter, body, orelse, .. }) => {
                ast::Statement::AsyncFor(ast::AsyncFor {
                    span: Span::new(l, r),
                    target,
                    iter,
                    body,
                    orelse,
                })
            }
            _ => unreachable!(),
        }
    },
};

IfStatement: ast::Statement = {
    <l:@L> "if" <t:Test> ":" <s:Suite> <e:ElseStatement?> <r:@R> => {
        ast::Statement::If(ast::If {
            span: Span::new(l, r),
            test: Box::new(t),
            body: s,
            orelse: e.unwrap_or_default(),
        })
    }
};

ElseStatement: Vec<ast::Statement> = {
    <l:@L> "elif" <t:Test> ":" <s:Suite> <e:ElseStatement?> <r:@R> => {
        vec![ast::Statement::If(ast::If {
            span: Span::new(l, r),
            test: Box::new(t),
            body: s,
            orelse: e.unwrap_or_default(),
        })]
    },
    "else" ":" <Suite>,
};

WhileStatement: ast::Statement = {
    <l:@L> "while" <t:Test> ":" <b:Suite> <e:("else" ":" <Suite>)?> <r:@R> => {
        ast::Statement::While(ast::While {
            span: Span::new(l, r),
            test: Box::new(t),
            body: b,
            orelse: e.unwrap_or_default(),
        })
    },
};

ForStatement: ast::Statement = {
    <l:@L> "for" <t:ExprList> "in" <i:TestList> ":" <b:Suite>
        <e:("else" ":" <Suite>)?> <r:@R>
    => {
        ast::Statement::For(ast::For {
            span: Span::new(l, r),
            target: Box::new(t),
            iter: Box::new(i),
            body: b,
            orelse: e.unwrap_or_default(),
        })
    },
};

TryStatement: ast::Statement = {
    <l:@L> "try" ":" <b:Suite> <c:ExceptClauseList>
        <e:("else" ":" <Suite>)?> <f:("finally" ":" <Suite>)?> <r:@R>
    => {
        ast::Statement::Try(ast::Try {
            span: Span::new(l, r),
            body: b,
            handlers: c,
            orelse: e.unwrap_or_default(),
            finalbody: f.unwrap_or_default(),
        })
    },
    <l:@L> "try" ":" <b:Suite> "finally" ":" <f:Suite> <r:@R> => {
        ast::Statement::Try(ast::Try {
            span: Span::new(l, r),
            body: b,
            handlers: vec![],
            orelse: vec![],
            finalbody: f,
        })
    },
};

ExceptClauseList: Vec<ast::ExceptHandler> = {
    <v:TypedExceptClause+> <c:DefaultExceptClause?> => {
        let mut v = v;
        if let Some(c) = c {
            v.push(c);
        }
        v
    },
    <DefaultExceptClause> => vec![<>],
};

TypedExceptClause: ast::ExceptHandler = {
    <l:@L> "except" <t:Test> <n:("as" <NAME>)?> ":" <s:Suite> <r:@R> => {
        ast::ExceptHandler {
            span: Span::new(l, r),
            typ: Some(Box::new(t)),
            name: n,
            body: s,
        }
    },
};

DefaultExceptClause: ast::ExceptHandler = {
    <l:@L> "except" ":" <s:Suite> <r:@R> => {
        ast::ExceptHandler {
            span: Span::new(l, r),
            typ: None,
            name: None,
            body: s,
        }
    }
};

WithStatement: ast::Statement = {
    <l:@L> "with" <v:Comma<WithItem>> ":" <s:Suite> <r:@R> => {
        ast::Statement::With(ast::With{
            span: Span::new(l, r),
            items: v,
            body: s,
        })
    },
};

WithItem: ast::WithItem = <t:Test> <e:("as" <Expr>)?> => {
    ast::WithItem {
        context_expr: Box::new(t),
        optional_vars: e.map(Box::new),
    }
};

ClassDef: ast::Statement = {
    <l:@L> "class" <n:NAME> <a:("(" <ArgList> ")")?> ":" <s:Suite> <r:@R> => {
        let (bases, keywords) = a.unwrap_or_default();
        ast::Statement::ClassDef(ast::ClassDef {
            span: Span::new(l, r),
            name: n,
            bases,
            keywords,
            body: s,
            decorator_list: vec![],
        })
    },
};

Suite: Vec<ast::Statement> = {
    SimpleStatement,
    NEWLINE INDENT <v:Statement+> DEDENT => v.concat(),
};

Test: ast::Expression = {Cond, OrTest, Lambda<Test>};

Cond: ast::Expression = {
    <l:@L> <t1:OrTest> "if" <t2:OrTest> "else" <t3:Test> <r:@R> => {
        ast::Expression::IfExp(ast::IfExp {
            span: Span::new(l, r),
            test: Box::new(t2),
            body: Box::new(t1),
            orelse: Box::new(t3),
        })
    },
};

TestNoCond: ast::Expression = {OrTest, Lambda<TestNoCond>};

Lambda<T>: ast::Expression = {
    <l:@L> "lambda" <v:UntypedParamList?> ":" <t:T> <r:@R> => {
        ast::Expression::Lambda(ast::Lambda {
            span: Span::new(l, r),
            args: Box::new(v.unwrap_or_default()),
            body: Box::new(t),
        })
    },
};

BoolExpr<O, E>: ast::Expression = <l:@L> <e:E> <v:(O E)*> => {
    v.into_iter().fold(e, |acc, (op, e)| {
        let r = e.span().end;
        ast::Expression::BoolOp(ast::BoolOp {
            span: Span::new(l, r),
            left: Box::new(acc),
            op,
            right: Box::new(e),
        })
    })
};

OrOp: ast::BoolOpKind = "or" => ast::BoolOpKind::Or;
AndOp: ast::BoolOpKind = "and" => ast::BoolOpKind::And;

OrTest: ast::Expression = BoolExpr<OrOp, AndTest>;
AndTest: ast::Expression = BoolExpr<AndOp, NotTest>;

NotTest: ast::Expression = {
    <l:@L> "not" <t:NotTest> <r:@R> => {
        ast::Expression::UnaryOp(ast::UnaryOp {
            span: Span::new(l, r),
            op: ast::UnaryOpKind::Not,
            operand: Box::new(t),
        })
    },
    Comparison,
};

CompOp: ast::ComparisonOpKind = {
    "<" => ast::ComparisonOpKind::Less,
    ">" => ast::ComparisonOpKind::Greater,
    "==" => ast::ComparisonOpKind::Equal,
    ">=" => ast::ComparisonOpKind::GreaterEqual,
    "<=" => ast::ComparisonOpKind::LessEqual,
    "!=" => ast::ComparisonOpKind::NotEqual,
    "in" => ast::ComparisonOpKind::In,
    "not" "in" => ast::ComparisonOpKind::NotIn,
    "is" "not" => ast::ComparisonOpKind::IsNot,
    "is" => ast::ComparisonOpKind::Is,
};

Comparison: ast::Expression = <l:@L> <e:Expr> <v:(CompOp Expr)*> <r:@R> => {
    if !v.is_empty() {
        let (ops, comparators) = v.into_iter().unzip();
        ast::Expression::Compare(ast::Compare {
            span: Span::new(l, r),
            left: Box::new(e),
            ops,
            comparators,
        })
    } else {
        e
    }
};

StarExpr: ast::Expression = <l:@L> "*" <e:Expr> <r:@R> => {
    ast::Expression::Starred(ast::Starred {
        span: Span::new(l, r),
        value: Box::new(e),
    })
};

BinExpr<O, E>: ast::Expression = <l:@L> <e:E> <v:(O E)*> => {
    v.into_iter().fold(e, |acc, (op, e)| {
        let r = e.span().end;
        ast::Expression::BinOp(ast::BinOp {
            span: Span::new(l, r),
            left: Box::new(acc),
            op,
            right: Box::new(e),
        })
    })
};

BitOrOp: ast::OpKind = "|" => ast::OpKind::BitOr;
Expr: ast::Expression = BinExpr<BitOrOp, XorExpr>;

XorOp: ast::OpKind = "^" => ast::OpKind::BitXor;
XorExpr: ast::Expression = BinExpr<XorOp, AndExpr>;

BitAndOp: ast::OpKind = "&" => ast::OpKind::BitAnd;
AndExpr: ast::Expression = BinExpr<BitAndOp, ShiftExpr>;

ShiftOp: ast::OpKind = {
    "<<" => ast::OpKind::LeftShift,
    ">>" => ast::OpKind::RightShift,
};
ShiftExpr: ast::Expression = BinExpr<ShiftOp, ArithExpr>;

AddSubOp: ast::OpKind = {
    "+" => ast::OpKind::Addition,
    "-" => ast::OpKind::Subtraction,
};
ArithExpr: ast::Expression = BinExpr<AddSubOp, Term>;

MulDivModOp: ast::OpKind = {
    "*" => ast::OpKind::Multiplication,
    "@" => ast::OpKind::MatrixMultiplication,
    "/" => ast::OpKind::Division,
    "%" => ast::OpKind::Modulo,
    "//" => ast::OpKind::FloorDivision,
};
Term: ast::Expression = BinExpr<MulDivModOp, Factor>;

UnaryOp: ast::UnaryOpKind = {
    "+" => ast::UnaryOpKind::Plus,
    "-" => ast::UnaryOpKind::Minus,
    "~" => ast::UnaryOpKind::Invert,
};

Factor: ast::Expression = {
    <l:@L> <op:UnaryOp> <f:Factor> <r:@R> => {
        ast::Expression::UnaryOp(ast::UnaryOp {
            span: Span::new(l, r),
            op,
            operand: Box::new(f),
        })
    },
    Power,
};

Power: ast::Expression = <l:@L> <e:AwaitExpr> <f:("**" <Factor>)?> <r:@R> => {
    match f {
        Some(factor) => {
            ast::Expression::BinOp(ast::BinOp {
                span: Span::new(l, r),
                left: Box::new(e),
                op: ast::OpKind::Power,
                right: Box::new(factor),
            })
        }
        _ => e,
    }
};

AwaitExpr: ast::Expression = <l:@L> <a:"await"?> <e:AtomExpr> <r:@R> => {
    if a.is_some() {
        ast::Expression::Await(ast::Await {
            span: Span::new(l, r),
            value: Box::new(e),
        })
    } else {
        e
    }
};

AtomExpr: ast::Expression = {
    <l:@L> <e:AtomExpr> "(" <a:ArgList?> ")" <r:@R> => {
        let (args, keywords) = a.unwrap_or_default();
        ast::Expression::Call(ast::Call {
            span: Span::new(l, r),
            func: Box::new(e),
            args,
            keywords,
        })
    },
    <l:@L> <e:AtomExpr> "[" <s:SubscriptList> "]" <r:@R> => {
        ast::Expression::Subscript(ast::Subscript {
            span: Span::new(l, r),
            value: Box::new(e),
            slice: s,
        })
    },
    <l:@L> <e:AtomExpr> "." <n:NAME> <r:@R> => {
        ast::Expression::Attribute(ast::Attribute {
            span: Span::new(l, r),
            value: Box::new(e),
            attr: n,
        })
    },
    Atom,
};

ArgList: (Vec<ast::Expression>, Vec<ast::Keyword>) = {
    <v:Comma<Arg>> ","? => {
        let mut args = vec![];
        let mut kwargs = vec![];
        for e in v {
            match e {
                (Some(arg), None) => args.push(arg),
                (None, Some(kw)) => kwargs.push(kw),
                _ => unreachable!(),
            }
        }
        (args, kwargs)
    },
    <GeneratorExp> => (vec![<>], vec![]),
};

Arg: (Option<ast::Expression>, Option<ast::Keyword>) = {
    <Test> => { (Some(<>), None) },
    <n:NAME> "=" <t:Test> => {
        let kw = ast::Keyword { arg: Some(n), value: Box::new(t) };
        (None, Some(kw))
    },
    <l:@L> "*" <t:Test> <r:@R> => {
        let arg = ast::Expression::Starred(ast::Starred {
            span: Span::new(l, r),
            value: Box::new(t),
        });
        (Some(arg), None)
    },
    "**" <Test> => { (None, Some(ast::Keyword { arg: None, value: Box::new(<>) })) },
};

SubscriptList: ast::Slice = <t:Subscript> <v:("," <Subscript>)*> <c:","?> => {
    if !v.is_empty() || c.is_some() {
        fn slice_index(slice: &ast::Slice) -> Option<&ast::Expression> {
            match slice {
                ast::Slice::Index { value } => Some(value),
                _ => None,
            }
        }
        if slice_index(&t).is_some() {
            if v.iter().map(slice_index).all(|x| x.is_some()) {
                let elts = iter::once(slice_index(&t))
                    .chain(v.iter().map(slice_index))
                    .map(|x| x.unwrap().clone())
                    .collect::<Vec<_>>();
                let tup = ast::Expression::Tuple(ast::Tuple {
                    span: Span::new(elts[0].span().start, elts.last().unwrap().span().end),
                    elts,
                });
                return ast::Slice::Index { value: Box::new(tup) }
            }
        }
        ast::Slice::ExtSlice { dims: iter::once(t).chain(v).collect() }
    } else {
        t
    }
};

Subscript: ast::Slice = {
    <Test> => ast::Slice::Index { value: Box::new(<>) },
    <e1:Test?> ":" <e2:Test?> <e3:(":" <Test?>)?> => {
        ast::Slice::Slice {
            lower: e1.map(Box::new),
            upper: e2.map(Box::new),
            step: e3.and_then(|x| x).map(Box::new),
        }
    },
};

Atom: ast::Expression = {
    "(" <e:YieldExpr> ")" => e,
    "(" <GeneratorExp> ")",
    "(" <TestListStarExpr> ")",
    <l:@L> "[" <t:Test> <c:Comprehension+> "]" <r:@R> => {
        ast::Expression::ListComp(ast::ListComp {
            span: Span::new(l, r),
            elt: Box::new(t),
            generators: c,
        })
    },
    <l:@L> "[" <v:Comma<TestOrStar>?> ","? "]" <r:@R> => {
        ast::Expression::List(ast::List {
            span: Span::new(l, r),
            elts: v.unwrap_or_default(),
        })
    },
    <l:@L> "{" <t:Test> <c:Comprehension+> "}" <r:@R> => {
        ast::Expression::SetComp(ast::SetComp {
            span: Span::new(l, r),
            elt: Box::new(t),
            generators: c,
        })
    },
    <l:@L> "{" <v:Comma<TestOrStar>> ","? "}" <r:@R> => {
        ast::Expression::Set(ast::Set {
            span: Span::new(l, r),
            elts: v,
        })
    },
    <l:@L> "{" <k:Test> ":" <v:Test> <c:Comprehension+> "}" <r:@R> => {
        ast::Expression::DictComp(ast::DictComp {
            span: Span::new(l, r),
            key: Box::new(k),
            value: Box::new(v),
            generators: c,
        })
    },
    <l:@L> "{" <v:Comma<KeyValue>?> ","? "}" <r:@R> => {
        let (keys, values) = v.unwrap_or_default().into_iter().unzip();
        ast::Expression::Dict(ast::Dict {
            span: Span::new(l, r),
            keys,
            values,
        })
    },
    <l:@L> <id:NAME> <r:@R> => ast::Expression::Name(ast::Name {
        span: Span::new(l, r),
        id,
    }),
    <l:@L> <n:Number> <r:@R> => ast::Expression::Num(ast::Num {
        span: Span::new(l, r),
        value: n,
    }),
    <l:@L> <v:BYTES+> <r:@R> => ast::Expression::Bytes(ast::Bytes {
        span: Span::new(l, r),
        value: v.concat(),
    }),
    <l:@L> <v:STRING+> <r:@R> => {
        ast::Expression::Str(ast::Str {
            span: Span::new(l, r),
            value: v.concat(),
        })
    },
    FSTRING+ => unimplemented!(),
    <l:@L> "..." <r:@R> => ast::Expression::Ellipsis(ast::Ellipsis {
        span: Span::new(l, r),
    }),
    <l:@L> <c:NameConstant> <r:@R> => {
        ast::Expression::NameConstant(ast::NameConstant {
            span: Span::new(l, r),
            value: c,
        })
    },
};

YieldExpr: ast::Expression = {
    <l:@L> "yield" "from" <t:Test> <r:@R> => {
        ast::Expression::YieldFrom(ast::YieldFrom {
            span: Span::new(l, r),
            value: Box::new(t),
        })
    },
    <l:@L> "yield" <t:TestList?> <r:@R> => {
        ast::Expression::Yield(ast::Yield {
            span: Span::new(l, r),
            value: t.map(Box::new),
        })
    },
};

Number: ast::NumKind = {
    INTEGER => ast::NumKind::Integer(<>),
    FLOAT => ast::NumKind::Float(<>),
    IMAGINARY => ast::NumKind::Complex(<>),
};

NameConstant: ast::Singleton = {
    "None" => ast::Singleton::None,
    "True" => ast::Singleton::True,
    "False" => ast::Singleton::False,
};

GeneratorExp: ast::Expression = <l:@L> <t:Test> <c:Comprehension+> <r:@R> => {
    ast::Expression::GeneratorExp(ast::GeneratorExp {
        span: Span::new(l, r),
        elt: Box::new(t),
        generators: c,
    })
};

Comprehension: ast::Comprehension = {
    <a:"async"?> "for" <e:ExprList> "in" <t:OrTest> <i:("if" <TestNoCond>)*> => {
        ast::Comprehension {
            target: Box::new(e),
            iter: Box::new(t),
            ifs: i,
            is_async: a.is_some(),
        }
    },
};

ExprOrStar: ast::Expression = {Expr, StarExpr};

ExprList: ast::Expression = SingleOrTuple<ExprOrStar>;

TestList: ast::Expression = SingleOrTuple<Test>;

TestOrStar: ast::Expression = {Test, StarExpr};

TestListStarExpr: ast::Expression = SingleOrTuple<TestOrStar>;

SingleOrTuple<T>: ast::Expression = {
    <l:@L> <t:T> <v:("," <T>)*> <c:","?> <r:@R> => {
        if !v.is_empty() || c.is_some() {
            let elts = iter::once(t).chain(v).collect();
            ast::Expression::Tuple(ast::Tuple {
                span: Span::new(l, r),
                elts,
            })
        } else {
            t
        }
    }
};

KeyValue: (Option<ast::Expression>, ast::Expression) = {
    <k:Test> ":" <v:Test> => (Some(k), v),
    "**" <Test> => (None, <>),
};

#[inline]
Sep<T, S>: Vec<T> =
    <e:T> <v:(S <T>)*> => iter::once(e).chain(v).collect();

#[inline]
Comma<T>: Vec<T> = Sep<T, ",">;

extern {
    type Location = Location;
    type Error = Error;

    enum Token {
        MODULE => Token::Module,
        INTERACTIVE => Token::Interactive,
        EVAL => Token::Eval,

        EOF => Token::Eof,
        NEWLINE => Token::Newline,
        INDENT => Token::Indent,
        DEDENT => Token::Dedent,
        NAME => Token::Name(<String>),
        INTEGER => Token::Integer(<num_bigint::BigUint>),
        FLOAT => Token::Float(<f64>),
        IMAGINARY => Token::Imaginary(<f64>),
        STRING => Token::String(<String>),
        FSTRING => Token::FormattedString(<String>),
        BYTES => Token::Bytes(<Vec<u8>>),

        "(" => Token::ParenOpen,
        ")" => Token::ParenClose,
        "[" => Token::BracketOpen,
        "]" => Token::BracketClose,
        "{" => Token::BraceOpen,
        "}" => Token::BraceClose,
        ":" => Token::Colon,
        "," => Token::Comma,
        ";" => Token::Semi,
        "+" => Token::Plus,
        "-" => Token::Minus,
        "*" => Token::Star,
        "/" => Token::Slash,
        "|" => Token::Pipe,
        "&" => Token::Amper,
        "<" => Token::Less,
        ">" => Token::Greater,
        "=" => Token::Equal,
        "." => Token::Dot,
        "%" => Token::Percent,
        "==" => Token::EqEqual,
        "!=" => Token::NotEqual,
        "<=" => Token::LessEqual,
        ">=" => Token::GreaterEqual,
        "~" => Token::Tilde,
        "^" => Token::Caret,
        "<<" => Token::LeftShift,
        ">>" => Token::RightShift,
        "**" => Token::DoubleStar,
        "+=" => Token::PlusEqual,
        "-=" => Token::MinEqual,
        "*=" => Token::StarEqual,
        "/=" => Token::SlashEqual,
        "%=" => Token::PercentEqual,
        "&=" => Token::AmperEqual,
        "|=" => Token::PipeEqual,
        "^=" => Token::CaretEqual,
        "<<=" => Token::LeftShiftEqual,
        ">>=" => Token::RightShiftEqual,
        "**=" => Token::DoubleStarEqual,
        "//" => Token::DoubleSlash,
        "//=" => Token::DoubleSlashEqual,
        "@" => Token::At,
        "@=" => Token::AtEqual,
        "->" => Token::Rarrow,
        "..." => Token::Ellipsis,

        "False" => Token::False,
        "None" => Token::None,
        "True" => Token::True,
        "and" => Token::And,
        "as" => Token::As,
        "assert" => Token::Assert,
        "break" => Token::Break,
        "class" => Token::Class,
        "continue" => Token::Continue,
        "def" => Token::Def,
        "del" => Token::Del,
        "elif" => Token::Elif,
        "else" => Token::Else,
        "except" => Token::Except,
        "finally" => Token::Finally,
        "for" => Token::For,
        "from" => Token::From,
        "global" => Token::Global,
        "if" => Token::If,
        "import" => Token::Import,
        "in" => Token::In,
        "is" => Token::Is,
        "lambda" => Token::Lambda,
        "nonlocal" => Token::Nonlocal,
        "not" => Token::Not,
        "or" => Token::Or,
        "pass" => Token::Pass,
        "raise" => Token::Raise,
        "return" => Token::Return,
        "try" => Token::Try,
        "while" => Token::While,
        "with" => Token::With,
        "yield" => Token::Yield,
        "await" => Token::Await,
        "async" => Token::Async,
    }
}