use crate::ast::*;
use crate::lexer::{lex, Tok, Token};
const KEYWORDS: &[&str] = &[
"False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue",
"def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import",
"in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while",
"with", "yield",
];
fn is_keyword(s: &str) -> bool {
KEYWORDS.contains(&s)
}
pub fn parse(src: &str) -> Result<Vec<Stmt>, String> {
let src = crate::rust_ffi::desugar(src);
let lexed = lex(&src)?;
let unclosed = lexed.unclosed;
let mut p = Parser {
toks: lexed.toks,
pos: 0,
deferred: lexed.deferred,
depth: 0,
in_function: false,
loop_depth: 0,
nesting: 0,
misplaced: None,
};
let err = match p.parse_module() {
Ok(stmts) => match unclosed {
Some(e) => e,
None => return Ok(stmts),
},
Err(e) => match unclosed {
Some(u) if p.pos + 2 >= p.toks.len() => u,
_ => e,
},
};
let lineno = split_syntax_error(&err).1.and_then(|p| p.lineno);
Err(match lineno.and_then(|l| source_line(&src, l, true)) {
Some(text) => with_text(err, &text),
None => err,
})
}
pub const SYNTAX_FIELD: char = '\u{1}';
#[derive(Debug, Clone, Default, PartialEq)]
pub struct SyntaxPos {
pub lineno: Option<i64>,
pub offset: Option<i64>,
pub end_lineno: Option<i64>,
pub end_offset: Option<i64>,
pub text: Option<String>,
pub filename: Option<String>,
pub no_source: bool,
pub bare_args: bool,
}
pub fn at_pos(msg: &str, lineno: u32, offset: i64, end_lineno: u32, end_offset: i64) -> String {
format!("{msg}{SYNTAX_FIELD}pos={lineno}:{offset}:{end_lineno}:{end_offset}")
}
pub fn with_text(err: String, text: &str) -> String {
let fields = err.find(SYNTAX_FIELD).map_or("", |i| &err[i..]);
if !fields.contains("\u{1}pos=")
|| fields.contains("\u{1}text=")
|| fields.contains("\u{1}nosrc=")
{
return err;
}
format!("{err}{SYNTAX_FIELD}text={text}")
}
pub fn with_filename(err: String, filename: &str) -> String {
let fields = err.find(SYNTAX_FIELD).map_or("", |i| &err[i..]);
if !fields.contains("\u{1}pos=") || fields.contains("\u{1}file=") {
return err;
}
format!("{err}{SYNTAX_FIELD}file={filename}")
}
pub fn split_syntax_error(err: &str) -> (&str, Option<SyntaxPos>) {
let Some(i) = err.find(SYNTAX_FIELD) else {
return (err, None);
};
let mut pos = SyntaxPos::default();
for field in err[i + 1..].split(SYNTAX_FIELD) {
let Some((key, value)) = field.split_once('=') else {
continue;
};
match key {
"pos" => {
let n: Vec<Option<i64>> = value.split(':').map(|s| s.parse().ok()).collect();
if let [l, o, el, eo] = n[..] {
pos.lineno = l;
pos.offset = o;
pos.end_lineno = el;
pos.end_offset = eo;
}
}
"text" => pos.text = Some(value.to_string()),
"nosrc" => pos.no_source = true,
"bare" => pos.bare_args = true,
"file" => pos.filename = Some(value.to_string()),
_ => {}
}
}
(&err[..i], Some(pos))
}
pub fn for_eval_input(err: String, src: &str) -> String {
let (head, Some(mut pos)) = split_syntax_error(&err) else {
return err;
};
let last_line = src.split_inclusive('\n').count() as i64;
let ends_without_newline = !src.ends_with('\n');
if let (Some(text), Some(l)) = (&pos.text, pos.lineno) {
if l == last_line && ends_without_newline {
let bare = text.trim_end_matches('\n').to_string();
let at_end = pos.offset == Some(bare.chars().count() as i64 + 1);
if at_end {
pos.offset = Some(0);
pos.end_offset = Some(0);
}
pos.text = Some(bare);
}
}
let mut out = head.to_string();
if let (Some(l), Some(o), Some(el), Some(eo)) =
(pos.lineno, pos.offset, pos.end_lineno, pos.end_offset)
{
out.push_str(&format!("{SYNTAX_FIELD}pos={l}:{o}:{el}:{eo}"));
}
if let Some(t) = &pos.text {
out.push_str(&format!("{SYNTAX_FIELD}text={t}"));
}
if let Some(f) = &pos.filename {
out.push_str(&format!("{SYNTAX_FIELD}file={f}"));
}
out
}
pub fn source_line(src: &str, lineno: i64, keep_newline: bool) -> Option<String> {
let idx = usize::try_from(lineno).ok()?.checked_sub(1)?;
let line = src.split_inclusive('\n').nth(idx)?;
Some(if keep_newline && !line.ends_with('\n') {
format!("{line}\n")
} else {
line.to_string()
})
}
pub fn render_syntax_block(pos: &SyntaxPos, default_file: &str) -> String {
let file = pos.filename.as_deref().unwrap_or(default_file);
let mut out = match pos.lineno {
Some(l) => format!(" File \"{file}\", line {l}\n"),
None => format!(" File \"{file}\"\n"),
};
let Some(text) = &pos.text else {
return out;
};
let rtext = text.trim_end_matches('\n');
let ltext = rtext.trim_start_matches([' ', '\n', '\x0c']);
let spaces = (rtext.chars().count() - ltext.chars().count()) as i64;
out.push_str(&format!(" {ltext}\n"));
let Some(mut offset) = pos.offset else {
return out;
};
let len = rtext.chars().count() as i64;
let mut end = if pos.lineno == pos.end_lineno {
match pos.end_offset {
Some(e) if e != 0 => e,
_ => offset,
}
} else {
len + 1
};
let text_len = text.chars().count() as i64;
if offset > text_len {
offset = len + 1;
}
if end > text_len {
end = len + 1;
}
if offset >= end || end < 0 {
end = offset + 1;
}
let col = offset - 1 - spaces;
let end_col = end - 1 - spaces;
if col >= 0 {
let lead: String = ltext
.chars()
.take(col as usize)
.map(|c| if c.is_whitespace() { c } else { ' ' })
.collect();
out.push_str(&format!(
" {lead}{}\n",
"^".repeat((end_col - col).max(0) as usize)
));
}
out
}
pub fn check_eval_input(src: &str) -> Result<(), String> {
let lexed = lex(src)?;
let mut p = Parser {
toks: lexed.toks,
pos: 0,
deferred: None,
depth: 0,
in_function: false,
loop_depth: 0,
nesting: 0,
misplaced: None,
};
p.skip_newlines();
if matches!(p.cur(), Tok::Eof) {
return Err(with_text(
at_pos("SyntaxError: invalid syntax", 0, 0, 0, 0),
"",
));
}
let checked = p.parse_exprlist().and_then(|_| {
p.skip_newlines();
if matches!(p.cur(), Tok::Eof) {
Ok(())
} else {
Err(p.err_here("invalid syntax"))
}
});
let lineno = checked
.as_ref()
.err()
.and_then(|e| split_syntax_error(e).1?.lineno);
checked.map_err(|e| match lineno.and_then(|l| source_line(src, l, false)) {
Some(text) => with_text(e, &text),
None => e,
})
}
const MAX_TREE_DEPTH: u32 = 20_000;
const TOO_COMPLEX: &str =
"MemoryError: Parser stack overflowed - Python source too complex to parse";
#[derive(Default)]
struct ArgOrder {
keyword: bool,
kw_unpack: bool,
}
impl ArgOrder {
fn positional(&self) -> Result<(), String> {
if self.kw_unpack {
Err("SyntaxError: positional argument follows keyword argument unpacking".into())
} else if self.keyword {
Err("SyntaxError: positional argument follows keyword argument".into())
} else {
Ok(())
}
}
fn star(&self) -> Result<(), String> {
if self.kw_unpack {
Err(
"SyntaxError: iterable argument unpacking follows keyword argument unpacking"
.into(),
)
} else {
Ok(())
}
}
}
struct Parser {
toks: Vec<Token>,
pos: usize,
deferred: Option<String>,
depth: u32,
in_function: bool,
loop_depth: u32,
nesting: u32,
misplaced: Option<String>,
}
fn spanned(e: Expr, line: u32, start: u32, end: u32, anchor_start: u32, anchor_end: u32) -> Expr {
Expr::Spanned(
Box::new(e),
Span {
line,
start,
end,
anchor_start,
anchor_end,
suppress: false,
},
)
}
impl Parser {
fn enter(&mut self) -> Result<(), String> {
self.depth += 1;
if self.depth > MAX_TREE_DEPTH {
return Err(TOO_COMPLEX.to_string());
}
Ok(())
}
fn cur(&self) -> &Tok {
&self.toks[self.pos].tok
}
fn line(&self) -> u32 {
self.toks[self.pos].line
}
fn col(&self) -> u32 {
self.toks[self.pos].col
}
fn cur_end_col(&self) -> u32 {
self.toks[self.pos].end_col
}
fn prev_end_col(&self) -> u32 {
self.toks[self.pos.saturating_sub(1)].end_col
}
fn advance(&mut self) -> Tok {
let t = self.toks[self.pos].tok.clone();
if self.pos + 1 < self.toks.len() {
self.pos += 1;
}
t
}
fn at_op(&self, s: &str) -> bool {
matches!(self.cur(), Tok::Op(o) if o == s)
}
fn eat_op(&mut self, s: &str) -> bool {
if self.at_op(s) {
self.advance();
true
} else {
false
}
}
fn expect_op(&mut self, s: &str) -> Result<(), String> {
if self.eat_op(s) {
Ok(())
} else {
Err(self.err_here(if s == ":" && self.at_newline() {
"expected ':'"
} else {
"invalid syntax"
}))
}
}
fn at_kw(&self, kw: &str) -> bool {
matches!(self.cur(), Tok::Name(n) if n == kw)
}
fn eat_kw(&mut self, kw: &str) -> bool {
if self.at_kw(kw) {
self.advance();
true
} else {
false
}
}
fn at_newline(&self) -> bool {
matches!(self.cur(), Tok::Newline)
}
fn skip_newlines(&mut self) {
while matches!(self.cur(), Tok::Newline) {
self.advance();
}
}
fn expect_name(&mut self) -> Result<String, String> {
match self.cur().clone() {
Tok::Name(n) if !is_keyword(&n) => {
self.advance();
Ok(n)
}
_ => Err(self.err_here("invalid syntax")),
}
}
fn err_here(&self, msg: &str) -> String {
self.err_span(msg, self.pos, self.pos)
}
fn err_span(&self, msg: &str, from: usize, to: usize) -> String {
self.err_span_as("SyntaxError", msg, from, to)
}
fn err_span_as(&self, class: &str, msg: &str, from: usize, to: usize) -> String {
let (line, offset, end_line, end_offset) = self.token_span(from, to);
at_pos(
&format!("{class}: {msg}"),
line,
offset as i64,
end_line,
end_offset as i64,
)
}
fn token_span(&self, from: usize, to: usize) -> (u32, u32, u32, u32) {
let a = &self.toks[from.min(self.toks.len() - 1)];
let b = &self.toks[to.min(self.toks.len() - 1)];
let end = match b.tok {
Tok::Newline | Tok::Eof | Tok::Indent | Tok::Dedent => b.col + 1,
_ => b.end_col,
};
(a.line, a.col + 1, b.line, end + 1)
}
fn span_suspension(&self, e: Expr, start: usize) -> Expr {
let (first, last) = (&self.toks[start], &self.toks[self.pos.saturating_sub(1)]);
if first.line != last.line {
return e;
}
spanned(e, first.line, first.col, last.end_col, 0, 0)
}
fn check_stmt_end(&self, stmt_start: usize) -> Result<(), String> {
if matches!(self.cur(), Tok::Newline | Tok::Eof | Tok::Dedent) || self.at_op(";") {
return Ok(());
}
let legacy = match &self.toks[stmt_start].tok {
Tok::Name(n) if self.pos == stmt_start + 1 && (n == "print" || n == "exec") => {
Some(n.clone())
}
_ => None,
};
if let (Some(n), true) = (legacy, self.starts_expression()) {
let mut last = self.pos;
while last + 1 < self.toks.len()
&& !matches!(self.toks[last + 1].tok, Tok::Newline | Tok::Eof)
{
last += 1;
}
return Err(self.err_span(
&format!("Missing parentheses in call to '{n}'. Did you mean {n}(...)?"),
stmt_start,
last,
));
}
Err(self.err_here("invalid syntax"))
}
fn comma_hint(&mut self, a_start: usize) -> Result<(), String> {
if !self.starts_expression() {
return Ok(());
}
let a = self.toks[a_start].tok.clone();
let single_name = self.pos == a_start + 1 && matches!(a, Tok::Name(_));
if single_name && matches!(self.cur(), Tok::Str(..)) {
return Ok(());
}
let legacy = match &a {
Tok::Name(n) if single_name && (n == "print" || n == "exec") => Some(n.clone()),
_ => None,
};
let b_start = self.pos;
let b_end = match self.parse_expr() {
Ok(_) => self.pos - 1,
Err(_) => b_start,
};
let msg = match legacy {
Some(n) => format!("Missing parentheses in call to '{n}'. Did you mean {n}(...)?"),
None => "invalid syntax. Perhaps you forgot a comma?".to_string(),
};
Err(self.err_span(&msg, a_start, b_end))
}
fn check_assign_targets(
&self,
targets: &[Expr],
spans: &[(usize, usize)],
) -> Result<(), String> {
let single = targets.len() == 1;
if let (true, Some(Expr::Tuple(items))) = (single, targets.first().map(unspan)) {
let (a, b) = spans[0];
let bare = !matches!(&self.toks[a].tok, Tok::Op(o) if o == "(");
if let (true, Some(last), Some(&(la, lb))) =
(bare, items.last(), self.split_commas(a, b).last())
{
let last = unspan(last);
let plain = !matches!(
last,
Expr::Name(_)
| Expr::Attribute(..)
| Expr::Subscript(..)
| Expr::Starred(_)
| Expr::Tuple(_)
| Expr::List(_)
| Expr::GenExp(..)
| Expr::True
| Expr::False
| Expr::None
| Expr::BoolOp(..)
| Expr::UnaryOp(UnOp::Not, _)
| Expr::Compare(..)
| Expr::IfExp { .. }
| Expr::Lambda { .. }
| Expr::NamedExpr(..)
);
if plain {
return Err(self.err_span(
&format!(
"cannot assign to {} here. Maybe you meant '==' instead of '='?",
expr_name(last)
),
la,
lb,
));
}
}
}
for (t, &(a, b)) in targets.iter().zip(spans) {
let Some((what, (ia, ib))) = self.invalid_target(t, a, b) else {
continue;
};
let e = unspan(t);
let comparable = !matches!(
e,
Expr::Tuple(_)
| Expr::List(_)
| Expr::GenExp(..)
| Expr::True
| Expr::False
| Expr::None
| Expr::BoolOp(..)
| Expr::UnaryOp(UnOp::Not, _)
| Expr::Compare(..)
| Expr::IfExp { .. }
| Expr::Lambda { .. }
| Expr::NamedExpr(..)
);
if matches!(e, Expr::Yield(_) | Expr::YieldFrom(_)) {
return Err(self.err_span("assignment to yield expression not possible", a, b));
}
let msg = if single && comparable {
format!("cannot assign to {what} here. Maybe you meant '==' instead of '='?")
} else {
format!("cannot assign to {what}")
};
return Err(self.err_span(&msg, ia, ib));
}
Ok(())
}
fn invalid_target(
&self,
e: &Expr,
a: usize,
b: usize,
) -> Option<(&'static str, (usize, usize))> {
match unspan(e) {
Expr::Name(_) | Expr::Attribute(..) | Expr::Subscript(..) => None,
Expr::Starred(inner) => self.invalid_target(inner, a + 1, b),
Expr::Tuple(items) | Expr::List(items) => {
let (ia, ib) = self.strip_brackets(a, b);
let parts = self.split_commas(ia, ib);
items
.iter()
.zip(parts)
.find_map(|(item, (x, y))| self.invalid_target(item, x, y))
}
other => Some((expr_name(other), (a, b))),
}
}
fn strip_brackets(&self, a: usize, b: usize) -> (usize, usize) {
let open = matches!(&self.toks[a].tok, Tok::Op(o) if o == "(" || o == "[");
let close = matches!(&self.toks[b].tok, Tok::Op(o) if o == ")" || o == "]");
if open && close && b > a + 1 && self.split_commas(a, b).len() == 1 {
(a + 1, b - 1)
} else {
(a, b)
}
}
fn split_commas(&self, a: usize, b: usize) -> Vec<(usize, usize)> {
let mut out = Vec::new();
let mut depth = 0i32;
let mut start = a;
for i in a..=b.min(self.toks.len() - 1) {
match &self.toks[i].tok {
Tok::Op(o) if matches!(o.as_str(), "(" | "[" | "{") => depth += 1,
Tok::Op(o) if matches!(o.as_str(), ")" | "]" | "}") => depth -= 1,
Tok::Op(o) if o == "," && depth == 0 => {
if i > start {
out.push((start, i - 1));
}
start = i + 1;
}
_ => {}
}
}
if start <= b {
out.push((start, b));
}
out
}
fn starts_expression(&self) -> bool {
match self.cur() {
Tok::Name(n) => {
!is_keyword(n)
|| matches!(
n.as_str(),
"None" | "True" | "False" | "not" | "lambda" | "await"
)
}
Tok::Op(o) => matches!(o.as_str(), "(" | "[" | "{" | "-" | "+" | "~" | "..."),
Tok::Newline | Tok::Indent | Tok::Dedent | Tok::Eof => false,
_ => true,
}
}
fn parse_module(&mut self) -> Result<Vec<Stmt>, String> {
let mut stmts = Vec::new();
self.skip_newlines();
while !matches!(self.cur(), Tok::Eof) {
self.parse_statement(&mut stmts)?;
self.skip_newlines();
}
match self.deferred.take().or_else(|| self.misplaced.take()) {
Some(e) => Err(e),
None => Ok(stmts),
}
}
fn note_misplaced(&mut self, msg: &str, start: usize) {
if self.misplaced.is_none() {
let e = self.err_span(msg, start, self.pos.saturating_sub(1));
self.misplaced = Some(format!("{e}{SYNTAX_FIELD}nosrc=1"));
}
}
fn parse_loop_body(&mut self, desc: &str, line: u32) -> Result<Vec<Stmt>, String> {
self.loop_depth += 1;
let body = self.parse_suite(desc, line);
self.loop_depth -= 1;
body
}
fn parse_scope_body(
&mut self,
desc: &str,
line: u32,
function: bool,
) -> Result<Vec<Stmt>, String> {
let saved = (self.in_function, self.loop_depth);
self.in_function = function;
self.loop_depth = 0;
self.nesting += 1;
let body = self.parse_suite(desc, line);
self.nesting -= 1;
(self.in_function, self.loop_depth) = saved;
body
}
fn parse_suite(&mut self, desc: &str, kw_line: u32) -> Result<Vec<Stmt>, String> {
let forced = matches!(
desc,
"function definition" | "'try' statement" | "'else' statement" | "'finally' statement"
);
if forced && !self.at_op(":") {
return Err(self.err_here("expected ':'"));
}
self.expect_op(":")?;
if self.at_newline() {
self.skip_newlines();
if !matches!(self.cur(), Tok::Indent) {
return Err(self.err_span_as(
"IndentationError",
&format!("expected an indented block after {desc} on line {kw_line}"),
self.pos,
self.pos,
));
}
self.advance(); let mut body = Vec::new();
while !matches!(self.cur(), Tok::Dedent | Tok::Eof) {
self.parse_statement(&mut body)?;
self.skip_newlines();
}
if matches!(self.cur(), Tok::Dedent) {
self.advance();
}
Ok(body)
} else {
let mut body = Vec::new();
self.parse_simple_line(&mut body)?;
Ok(body)
}
}
fn parse_statement(&mut self, out: &mut Vec<Stmt>) -> Result<(), String> {
let line = self.line();
if matches!(self.cur(), Tok::Indent) {
let first = &self.toks[(self.pos + 1).min(self.toks.len() - 1)];
return Err(at_pos(
"IndentationError: unexpected indent",
first.line,
first.col as i64,
first.line,
-1,
));
}
if let Tok::Name(n) = self.cur().clone() {
match n.as_str() {
"if" => return self.parse_if(out, line),
"while" => return self.parse_while(out, line),
"for" => return self.parse_for(out, line, false),
"def" => return self.parse_funcdef(out, line, Vec::new(), false),
"class" => return self.parse_classdef(out, line, Vec::new()),
"try" => return self.parse_try(out, line),
"with" => return self.parse_with(out, line, false),
"async" => return self.parse_async(out, line),
"match" if self.looks_like_match() => return self.parse_match(out, line),
_ => {}
}
}
if self.at_op("@") {
return self.parse_decorated(out, line);
}
if self.looks_like_stray_case() {
return Err("SyntaxError: invalid syntax. Did you mean 'class'?".to_string());
}
self.parse_simple_line(out)
}
fn looks_like_stray_case(&self) -> bool {
matches!(self.cur(), Tok::Name(n) if n == "case")
&& matches!(
self.toks.get(self.pos + 1).map(|t| &t.tok),
Some(Tok::Name(_))
)
&& matches!(
self.toks.get(self.pos + 2).map(|t| &t.tok),
Some(Tok::Op(o)) if o == ":"
)
}
fn parse_simple_line(&mut self, out: &mut Vec<Stmt>) -> Result<(), String> {
loop {
let stmt_start = self.pos;
self.parse_simple_stmt(out)?;
self.check_stmt_end(stmt_start)?;
if self.eat_op(";") {
if self.at_newline() || matches!(self.cur(), Tok::Eof) {
break;
}
continue;
}
break;
}
if self.at_newline() {
self.advance();
}
Ok(())
}
fn parse_simple_stmt(&mut self, out: &mut Vec<Stmt>) -> Result<(), String> {
let line = self.line();
if let Tok::Name(n) = self.cur().clone() {
match n.as_str() {
"pass" => {
self.advance();
out.push(Stmt::new(StmtKind::Pass, line));
return Ok(());
}
"break" => {
let start = self.pos;
self.advance();
if self.loop_depth == 0 {
self.note_misplaced("'break' outside loop", start);
}
out.push(Stmt::new(StmtKind::Break, line));
return Ok(());
}
"continue" => {
let start = self.pos;
self.advance();
if self.loop_depth == 0 {
self.note_misplaced("'continue' not properly in loop", start);
}
out.push(Stmt::new(StmtKind::Continue, line));
return Ok(());
}
"return" => {
let start = self.pos;
self.advance();
let v =
if self.at_newline() || self.at_op(";") || matches!(self.cur(), Tok::Eof) {
None
} else {
Some(self.parse_exprlist()?)
};
if !self.in_function {
self.note_misplaced("'return' outside function", start);
}
out.push(Stmt::new(StmtKind::Return(v), line));
return Ok(());
}
"raise" => return self.parse_raise(out, line),
"del" => {
self.advance();
let start = self.pos;
let targets = self.parse_target_list()?;
let items = self.split_commas(start, self.pos - 1);
for (t, (a, b)) in targets.iter().zip(items) {
if let Some((what, (a, b))) = self.invalid_target(t, a, b) {
return Err(self.err_span(&format!("cannot delete {what}"), a, b));
}
}
out.push(Stmt::new(StmtKind::Delete(targets), line));
return Ok(());
}
"assert" => {
self.advance();
let test = self.parse_expr()?;
let msg = if self.eat_op(",") {
Some(self.parse_expr()?)
} else {
None
};
out.push(Stmt::new(StmtKind::Assert { test, msg }, line));
return Ok(());
}
"global" => {
let start = self.pos;
self.advance();
let names = self.parse_name_list()?;
let mut stmt = Stmt::new(StmtKind::Global(names), line);
stmt.span = Some(self.token_span(start, self.pos - 1));
out.push(stmt);
return Ok(());
}
"nonlocal" => {
let start = self.pos;
self.advance();
let names = self.parse_name_list()?;
if self.nesting == 0 && self.misplaced.is_none() {
self.note_misplaced(
"nonlocal declaration not allowed at module level",
start,
);
if let Some(e) = &mut self.misplaced {
e.push_str(&format!("{SYNTAX_FIELD}bare=1"));
}
}
let mut stmt = Stmt::new(StmtKind::Nonlocal(names), line);
stmt.span = Some(self.token_span(start, self.pos - 1));
out.push(stmt);
return Ok(());
}
"import" => return self.parse_import(out, line),
"from" => return self.parse_from_import(out, line),
_ => {}
}
}
self.parse_expr_stmt(out, line)
}
fn parse_name_list(&mut self) -> Result<Vec<String>, String> {
let mut names = vec![self.expect_name()?];
while self.eat_op(",") {
names.push(self.expect_name()?);
}
Ok(names)
}
fn parse_target_list(&mut self) -> Result<Vec<Expr>, String> {
let mut ts = vec![self.parse_expr()?];
while self.eat_op(",") {
if self.at_newline() || matches!(self.cur(), Tok::Eof) {
break;
}
ts.push(self.parse_expr()?);
}
Ok(ts)
}
fn parse_expr_stmt(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let first_start = self.pos;
let first = self.parse_exprlist()?;
if matches!(first, Expr::NamedExpr(..))
&& matches!(self.toks[first_start].tok, Tok::Name(_))
{
return Err(self.err_span("invalid syntax", first_start + 1, first_start + 1));
}
if self.at_op(":") {
self.advance();
let annotation = self.parse_expr()?;
let value = if self.eat_op("=") {
Some(self.parse_exprlist()?)
} else {
None
};
out.push(Stmt::new(
StmtKind::AnnAssign {
target: first,
annotation,
value,
},
line,
));
return Ok(());
}
let first_end = self.pos.saturating_sub(1);
if let Tok::Op(o) = self.cur().clone() {
if let Some(op) = augassign_op(&o) {
let target = unspan(&first);
if !matches!(
target,
Expr::Name(_) | Expr::Attribute(..) | Expr::Subscript(..)
) {
return Err(self.err_span(
&format!(
"'{}' is an illegal expression for augmented assignment",
expr_name(target)
),
first_start,
first_end,
));
}
self.advance();
let value = self.parse_exprlist()?;
out.push(Stmt::new(
StmtKind::AugAssign {
target: first,
op,
value,
},
line,
));
return Ok(());
}
}
if self.at_op("=") {
let mut targets = vec![first];
let mut spans = vec![(first_start, first_end)];
let mut value = None;
while self.eat_op("=") {
let start = self.pos;
let e = self.parse_exprlist()?;
if let Some(prev) = value.take() {
targets.push(prev);
}
spans.push((start, self.pos - 1));
value = Some(e);
}
self.check_assign_targets(&targets, &spans)?;
out.push(Stmt::new(
StmtKind::Assign {
targets,
value: value.unwrap(),
},
line,
));
return Ok(());
}
out.push(Stmt::new(StmtKind::Expr(first), line));
Ok(())
}
fn parse_if(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let kw = if matches!(self.cur(), Tok::Name(n) if n == "elif") {
"'elif' statement"
} else {
"'if' statement"
};
self.advance(); let test = self.parse_namedexpr()?;
let body = self.parse_suite(kw, line)?;
let mut orelse = Vec::new();
self.skip_newlines_shallow();
if self.at_kw("elif") {
self.parse_if(&mut orelse, self.line())?;
} else if self.at_kw("else") {
let else_line = self.line();
self.advance();
orelse = self.parse_suite("'else' statement", else_line)?;
}
out.push(Stmt::new(StmtKind::If { test, body, orelse }, line));
Ok(())
}
fn skip_newlines_shallow(&mut self) {
}
fn parse_while(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance();
let test = self.parse_namedexpr()?;
let body = self.parse_loop_body("'while' statement", line)?;
let orelse = if self.at_kw("else") {
let el = self.line();
self.advance();
self.parse_suite("'else' statement", el)?
} else {
Vec::new()
};
out.push(Stmt::new(StmtKind::While { test, body, orelse }, line));
Ok(())
}
fn parse_for(&mut self, out: &mut Vec<Stmt>, line: u32, is_async: bool) -> Result<(), String> {
self.advance();
let target = self.parse_target_tuple()?;
if !self.eat_kw("in") {
return Err(format!(
"SyntaxError: expected 'in' in for (line {})",
self.line()
));
}
let iter = self.parse_exprlist()?;
let body = self.parse_loop_body("'for' statement", line)?;
let orelse = if self.at_kw("else") {
let el = self.line();
self.advance();
self.parse_suite("'else' statement", el)?
} else {
Vec::new()
};
out.push(Stmt::new(
StmtKind::For {
target,
iter,
body,
orelse,
is_async,
},
line,
));
Ok(())
}
fn parse_target_tuple(&mut self) -> Result<Expr, String> {
let first = self.parse_target_atom()?;
if self.at_op(",") {
let mut items = vec![first];
while self.eat_op(",") {
if self.at_kw("in") || self.at_op("=") || self.at_op(":") {
break;
}
items.push(self.parse_target_atom()?);
}
Ok(Expr::Tuple(items))
} else {
Ok(first)
}
}
fn parse_target_atom(&mut self) -> Result<Expr, String> {
if self.eat_op("*") {
return Ok(Expr::Starred(Box::new(self.parse_await_postfix()?)));
}
self.parse_await_postfix()
}
fn parenthesized_with_items(&mut self) -> Option<Vec<WithItem>> {
if !self.at_op("(") {
return None;
}
let save = self.pos;
self.advance();
let mut items = Vec::new();
let shaped = loop {
if self.at_op(")") {
break !items.is_empty();
}
let Ok(context) = self.parse_expr() else {
break false;
};
let vars = if self.eat_kw("as") {
match self.parse_ternary() {
Ok(v) => Some(v),
Err(_) => break false,
}
} else {
None
};
items.push(WithItem { context, vars });
if !self.eat_op(",") {
break true;
}
};
if shaped && self.eat_op(")") && self.at_op(":") {
return Some(items);
}
self.pos = save;
None
}
fn parse_with(&mut self, out: &mut Vec<Stmt>, line: u32, is_async: bool) -> Result<(), String> {
self.advance();
let items = match self.parenthesized_with_items() {
Some(items) => items,
None => {
let mut items = Vec::new();
loop {
let context = self.parse_expr()?;
let vars = if self.eat_kw("as") {
Some(self.parse_ternary()?)
} else {
None
};
items.push(WithItem { context, vars });
if !self.eat_op(",") {
break;
}
}
items
}
};
let body = self.parse_suite("'with' statement", line)?;
out.push(Stmt::new(
StmtKind::With {
items,
body,
is_async,
},
line,
));
Ok(())
}
fn parse_async(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance(); if self.at_kw("def") {
return self.parse_funcdef(out, line, Vec::new(), true);
}
if self.at_kw("for") {
return self.parse_for(out, line, true);
}
if self.at_kw("with") {
return self.parse_with(out, line, true);
}
let _ = line;
Err(self.err_here("invalid syntax"))
}
fn parse_decorated(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let mut decorators = Vec::new();
while self.eat_op("@") {
decorators.push(self.parse_namedexpr()?);
if self.at_newline() {
self.advance();
}
self.skip_newlines();
}
if self.eat_kw("async") {
return self.parse_funcdef(out, line, decorators, true);
}
if self.at_kw("def") {
self.parse_funcdef(out, line, decorators, false)
} else if self.at_kw("class") {
self.parse_classdef(out, line, decorators)
} else {
{
let _ = line;
Err("SyntaxError: invalid syntax".to_string())
}
}
}
fn parse_type_params(&mut self) -> Result<Vec<String>, String> {
let mut names = Vec::new();
if !self.at_op("[") {
return Ok(names);
}
self.advance(); let mut depth = 1usize;
let mut expect_name = true;
loop {
match self.cur().clone() {
Tok::Op(o) if o == "[" => {
depth += 1;
expect_name = false;
}
Tok::Op(o) if o == "]" => {
depth -= 1;
if depth == 0 {
self.advance();
return Ok(names);
}
}
Tok::Op(o) if depth == 1 && o == "," => expect_name = true,
Tok::Op(o) if depth == 1 && (o == "*" || o == "**") => {}
Tok::Op(_) if depth == 1 => expect_name = false, Tok::Name(n) if depth == 1 && expect_name => {
names.push(n);
expect_name = false;
}
Tok::Eof => return Err("SyntaxError: unterminated type-parameter list".to_string()),
_ => {}
}
self.advance();
}
}
fn bind_type_params(&self, out: &mut Vec<Stmt>, params: &[String], line: u32) {
for name in params {
out.push(Stmt::new(
StmtKind::Assign {
targets: vec![Expr::Name(name.clone())],
value: Expr::Name("object".to_string()),
},
line,
));
}
}
fn parse_funcdef(
&mut self,
out: &mut Vec<Stmt>,
line: u32,
decorators: Vec<Expr>,
is_async: bool,
) -> Result<(), String> {
self.advance(); let name = self.expect_name()?;
let type_params = self.parse_type_params()?; self.bind_type_params(out, &type_params, line);
self.expect_op("(")?;
let mut params = self.parse_params(")")?;
self.expect_op(")")?;
if self.eat_op("->") {
let ret = self.parse_expr()?; params.annotations.push(("return".to_string(), ret));
}
let body = self.parse_scope_body("function definition", line, true)?;
out.push(Stmt::new(
StmtKind::FuncDef {
name,
params,
body,
decorators,
is_async,
},
line,
));
Ok(())
}
fn parse_params(&mut self, close: &str) -> Result<Params, String> {
let mut p = Params::default();
let mut seen_star = false;
let mut seen_default = false;
let mut seen_kwargs = false;
let mut names_seen: Vec<String> = Vec::new();
let mut check_dup = |parser: &mut Self, name: &str, at: usize| {
if names_seen.iter().any(|n| n == name) {
if parser.misplaced.is_none() {
let e = parser.err_span(
&format!("duplicate argument '{name}' in function definition"),
at,
at,
);
parser.misplaced =
Some(format!("{e}{SYNTAX_FIELD}nosrc=1{SYNTAX_FIELD}bare=1"));
}
} else {
names_seen.push(name.to_string());
}
};
loop {
if self.at_op(close) {
break;
}
if seen_kwargs {
return Err(self.err_here("arguments cannot follow var-keyword argument"));
}
if self.eat_op("/") {
p.posonly = p.names.len();
let _ = self.eat_op(",");
continue;
}
if self.at_op("*") {
if seen_star {
return Err(self.err_here("* argument may appear only once"));
}
let star_at = self.pos;
self.advance();
if self.at_op(close)
|| self.at_op(",")
&& matches!(&self.toks[self.pos + 1].tok, Tok::Op(o) if o == close || o == "**")
{
let at = if close == ")" { star_at } else { self.pos };
return Err(self.err_span("named arguments must follow bare *", at, at));
}
if self.at_op(",") {
p.star = Some(String::new()); } else {
let name_at = self.pos;
let star_name = self.expect_name()?;
check_dup(self, &star_name, name_at);
if close == ")" && self.at_op(":") {
self.advance();
let ann = self.parse_expr()?;
p.annotations.push((star_name.clone(), ann));
}
p.star = Some(star_name);
}
seen_star = true;
let _ = self.eat_op(",");
continue;
}
if self.eat_op("**") {
let name_at = self.pos;
let kw_name = self.expect_name()?;
check_dup(self, &kw_name, name_at);
seen_kwargs = true;
if close == ")" && self.at_op(":") {
self.advance();
let ann = self.parse_expr()?;
p.annotations.push((kw_name.clone(), ann));
}
p.kwargs = Some(kw_name);
let _ = self.eat_op(",");
continue;
}
let name_at = self.pos;
let name = self.expect_name()?;
check_dup(self, &name, name_at);
if close == ")" && self.eat_op(":") {
let ann = self.parse_expr()?;
p.annotations.push((name.clone(), ann));
}
let default = if self.eat_op("=") {
Some(self.parse_expr()?)
} else {
None
};
if default.is_some() {
seen_default = !seen_star;
} else if seen_default && !seen_star {
return Err(self.err_span(
"parameter without a default follows parameter with a default",
name_at,
name_at,
));
}
if seen_star {
p.kwonly.push(name);
p.kwonly_defaults.push(default);
} else {
p.names.push(name);
if let Some(d) = default {
p.defaults.push(d);
}
}
if !self.eat_op(",") {
break;
}
}
Ok(p)
}
fn parse_classdef(
&mut self,
out: &mut Vec<Stmt>,
line: u32,
decorators: Vec<Expr>,
) -> Result<(), String> {
self.advance(); let name = self.expect_name()?;
let type_params = self.parse_type_params()?; self.bind_type_params(out, &type_params, line);
let mut bases = Vec::new();
let mut keywords = Vec::new();
if self.eat_op("(") {
let mut order = ArgOrder::default();
while !self.at_op(")") {
if self.eat_op("**") {
order.kw_unpack = true;
keywords.push(Keyword {
name: None,
value: self.parse_expr()?,
});
} else if matches!(self.cur(), Tok::Name(n) if !is_keyword(n))
&& matches!(&self.toks[self.pos + 1].tok, Tok::Op(o) if o == "=")
{
order.keyword = true;
let kn = self.expect_name()?;
self.expect_op("=")?;
keywords.push(Keyword {
name: Some(kn),
value: self.parse_expr()?,
});
} else {
order.positional()?;
bases.push(self.parse_expr()?);
}
if !self.eat_op(",") {
break;
}
}
self.expect_op(")")?;
}
let body = self.parse_scope_body("class definition", line, false)?;
out.push(Stmt::new(
StmtKind::ClassDef {
name,
bases,
keywords,
body,
decorators,
},
line,
));
Ok(())
}
fn parse_try(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance();
let body = self.parse_suite("'try' statement", line)?;
let mut handlers = Vec::new();
while self.at_kw("except") {
let except_line = self.line();
self.advance();
let star = self.eat_op("*");
let (typ, name) = if self.at_op(":") {
(None, None)
} else {
let start = self.pos;
let mut t = self.parse_expr()?;
if self.at_op(",") {
let mut types = vec![t];
while self.eat_op(",") && !self.at_op(":") && !self.at_kw("as") {
types.push(self.parse_expr()?);
}
if self.eat_kw("as") {
self.expect_name()?;
return Err(self.err_span(
"multiple exception types must be parenthesized when using 'as'",
start,
self.pos - 1,
));
}
t = Expr::Tuple(types);
}
let n = if self.eat_kw("as") {
Some(self.expect_name()?)
} else {
None
};
(Some(t), n)
};
let hbody = self.parse_suite("'except' statement", except_line)?;
handlers.push(ExceptHandler {
typ,
name,
body: hbody,
star,
});
}
if handlers.is_empty() && !self.at_kw("finally") {
const MSG: &str = "expected 'except' or 'finally' block";
if matches!(self.cur(), Tok::Eof | Tok::Dedent) {
if let Some(nl) = self.toks[..self.pos]
.iter()
.rev()
.find(|t| matches!(t.tok, Tok::Newline))
{
return Err(at_pos(
&format!("SyntaxError: {MSG}"),
nl.line,
nl.col as i64 + 1,
nl.line,
-1,
));
}
}
return Err(self.err_here(MSG));
}
let orelse = if self.at_kw("else") {
let el = self.line();
self.advance();
self.parse_suite("'else' statement", el)?
} else {
Vec::new()
};
let finalbody = if self.at_kw("finally") {
let fl = self.line();
self.advance();
self.parse_suite("'finally' statement", fl)?
} else {
Vec::new()
};
out.push(Stmt::new(
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
},
line,
));
Ok(())
}
fn looks_like_match(&self) -> bool {
let mut i = self.pos + 1;
match self.toks.get(i).map(|t| &t.tok) {
Some(Tok::Op(o)) if o == "=" || o == ":" || o == "." || o == ";" || o == "," => {
return false
}
Some(Tok::Newline) | Some(Tok::Eof) | None => return false,
_ => {}
}
let mut depth = 0i32;
while let Some(t) = self.toks.get(i) {
match &t.tok {
Tok::Op(o) if o == "(" || o == "[" || o == "{" => depth += 1,
Tok::Op(o) if o == ")" || o == "]" || o == "}" => depth -= 1,
Tok::Op(o) if o == ":" && depth == 0 => {
let a1 = self.toks.get(i + 1).map(|t| &t.tok);
let a2 = self.toks.get(i + 2).map(|t| &t.tok);
let real_block = matches!(a1, Some(Tok::Newline))
&& matches!(a2, Some(Tok::Indent))
&& matches!(self.toks.get(i + 3).map(|t| &t.tok), Some(Tok::Name(n)) if n == "case");
let bare_header = matches!(a1, Some(Tok::Eof))
|| (matches!(a1, Some(Tok::Newline)) && !matches!(a2, Some(Tok::Indent)));
return real_block || bare_header;
}
Tok::Newline | Tok::Eof => return false,
_ => {}
}
i += 1;
}
false
}
fn parse_match(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance(); let subject = self.parse_exprlist()?;
self.expect_op(":")?;
self.skip_newlines();
if !matches!(self.cur(), Tok::Indent) {
return Err(format!(
"IndentationError: expected an indented block after 'match' statement on line {line}"
));
}
self.advance(); let mut cases = Vec::new();
while self.at_kw("case") {
let case_line = self.line();
self.advance(); let pattern = self.parse_patterns()?;
let guard = if self.eat_kw("if") {
Some(self.parse_namedexpr()?)
} else {
None
};
let body = self.parse_suite("'case' statement", case_line)?;
self.skip_newlines();
cases.push(MatchCase {
pattern,
guard,
body,
});
}
if matches!(self.cur(), Tok::Dedent) {
self.advance();
}
out.push(Stmt::new(StmtKind::Match { subject, cases }, line));
Ok(())
}
fn parse_patterns(&mut self) -> Result<Pattern, String> {
let first = self.parse_pattern()?;
if self.at_op(",") {
let mut elems = vec![first];
while self.eat_op(",") {
if self.at_op(":") || self.at_kw("if") {
break;
}
elems.push(self.parse_pattern()?);
}
let star = elems.iter().position(|p| matches!(p, Pattern::Star(_)));
Ok(Pattern::Sequence { elems, star })
} else {
Ok(first)
}
}
fn parse_pattern(&mut self) -> Result<Pattern, String> {
let p = self.parse_or_pattern()?;
if self.eat_kw("as") {
let name = self.expect_name()?;
Ok(Pattern::As(Box::new(p), name))
} else {
Ok(p)
}
}
fn parse_or_pattern(&mut self) -> Result<Pattern, String> {
let first = self.parse_closed_pattern()?;
if self.at_op("|") {
let mut alts = vec![first];
while self.eat_op("|") {
alts.push(self.parse_closed_pattern()?);
}
Ok(Pattern::Or(alts))
} else {
Ok(first)
}
}
fn parse_closed_pattern(&mut self) -> Result<Pattern, String> {
if self.eat_op("*") {
let name = if self.at_kw("_") {
self.advance();
None
} else {
Some(self.expect_name()?)
};
return Ok(Pattern::Star(name));
}
if self.eat_op("[") {
return self.parse_sequence_pattern("]");
}
if self.at_op("(") {
self.advance();
if self.eat_op(")") {
return Ok(Pattern::Sequence {
elems: vec![],
star: None,
});
}
let first = self.parse_pattern()?;
if self.at_op(",") {
let mut elems = vec![first];
while self.eat_op(",") {
if self.at_op(")") {
break;
}
elems.push(self.parse_pattern()?);
}
self.expect_op(")")?;
let star = elems.iter().position(|p| matches!(p, Pattern::Star(_)));
return Ok(Pattern::Sequence { elems, star });
}
self.expect_op(")")?;
return Ok(first);
}
if self.at_op("{") {
return self.parse_mapping_pattern();
}
if let Some(p) = self.try_literal_pattern()? {
return Ok(p);
}
let name = self.expect_name()?;
if name == "_" {
return Ok(Pattern::Wildcard);
}
let mut expr = Expr::Name(name);
let mut dotted = false;
while self.eat_op(".") {
let attr = self.expect_name()?;
expr = Expr::Attribute(Box::new(expr), attr);
dotted = true;
}
if self.at_op("(") {
return self.parse_class_pattern(expr);
}
if dotted {
Ok(Pattern::Value(expr))
} else {
match expr {
Expr::Name(n) => Ok(Pattern::Capture(n)),
_ => Ok(Pattern::Value(expr)),
}
}
}
fn try_literal_pattern(&mut self) -> Result<Option<Pattern>, String> {
if self.at_op("-") {
self.advance();
let e = self.parse_atom()?;
return Ok(Some(Pattern::Value(Expr::UnaryOp(UnOp::Neg, Box::new(e)))));
}
let lit = match self.cur().clone() {
Tok::Int(_)
| Tok::BigInt(_)
| Tok::Float(_)
| Tok::Complex(_)
| Tok::Str(_)
| Tok::FString(_, _)
| Tok::TString(_, _)
| Tok::Bytes(_) => Some(self.parse_atom()?),
Tok::Name(n) if n == "None" || n == "True" || n == "False" => Some(self.parse_atom()?),
_ => None,
};
Ok(lit.map(Pattern::Value))
}
fn parse_sequence_pattern(&mut self, close: &str) -> Result<Pattern, String> {
let mut elems = Vec::new();
while !self.at_op(close) {
elems.push(self.parse_pattern()?);
if !self.eat_op(",") {
break;
}
}
self.expect_op(close)?;
let star = elems.iter().position(|p| matches!(p, Pattern::Star(_)));
Ok(Pattern::Sequence { elems, star })
}
fn parse_mapping_pattern(&mut self) -> Result<Pattern, String> {
self.advance(); let mut keys = Vec::new();
let mut rest = None;
while !self.at_op("}") {
if self.eat_op("**") {
rest = Some(self.expect_name()?);
let _ = self.eat_op(",");
break;
}
let key = self.parse_or()?;
self.expect_op(":")?;
let pat = self.parse_pattern()?;
keys.push((key, pat));
if !self.eat_op(",") {
break;
}
}
self.expect_op("}")?;
Ok(Pattern::Mapping { keys, rest })
}
fn parse_class_pattern(&mut self, cls: Expr) -> Result<Pattern, String> {
self.expect_op("(")?;
let mut pos = Vec::new();
let mut kw = Vec::new();
while !self.at_op(")") {
if matches!(self.cur(), Tok::Name(n) if !is_keyword(n))
&& matches!(&self.toks[self.pos + 1].tok, Tok::Op(o) if o == "=")
{
let kn = self.expect_name()?;
self.expect_op("=")?;
kw.push((kn, self.parse_pattern()?));
} else {
pos.push(self.parse_pattern()?);
}
if !self.eat_op(",") {
break;
}
}
self.expect_op(")")?;
Ok(Pattern::Class { cls, pos, kw })
}
fn parse_raise(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance();
let (exc, cause) = if self.at_newline() || self.at_op(";") || matches!(self.cur(), Tok::Eof)
{
(None, None)
} else {
let e = self.parse_expr()?;
let c = if self.eat_kw("from") {
Some(self.parse_expr()?)
} else {
None
};
(Some(e), c)
};
out.push(Stmt::new(StmtKind::Raise { exc, cause }, line));
Ok(())
}
fn parse_import(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance();
self.names_after_import()?;
let mut names = Vec::new();
loop {
let mut name = self.expect_name()?;
while self.eat_op(".") {
name.push('.');
name.push_str(&self.expect_name()?);
}
let asname = if self.eat_kw("as") {
Some(self.expect_name()?)
} else {
None
};
names.push(Alias { name, asname });
if !self.eat_op(",") {
break;
}
}
out.push(Stmt::new(StmtKind::Import(names), line));
Ok(())
}
fn names_after_import(&self) -> Result<(), String> {
if !matches!(self.cur(), Tok::Newline | Tok::Eof) {
return Ok(());
}
let t = &self.toks[self.pos];
Err(at_pos(
"SyntaxError: Expected one or more names after 'import'",
t.line,
t.col as i64 + 1,
t.line,
t.col as i64 + 1,
))
}
fn parse_from_import(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance(); let mut level = 0;
while self.at_op(".") || self.at_op("...") {
level += if self.at_op("...") { 3 } else { 1 };
self.advance();
}
let module = if self.at_kw("import") {
None
} else {
let mut m = self.expect_name()?;
while self.eat_op(".") {
m.push('.');
m.push_str(&self.expect_name()?);
}
Some(m)
};
if !self.eat_kw("import") {
return Err(self.err_here("invalid syntax"));
}
self.names_after_import()?;
let mut names = Vec::new();
if self.eat_op("*") {
names.push(Alias {
name: "*".into(),
asname: None,
});
} else {
let paren = self.eat_op("(");
loop {
let name = self.expect_name()?;
let asname = if self.eat_kw("as") {
Some(self.expect_name()?)
} else {
None
};
names.push(Alias { name, asname });
if !self.eat_op(",") {
break;
}
if paren && self.at_op(")") {
break;
}
}
if paren {
self.expect_op(")")?;
}
}
out.push(Stmt::new(
StmtKind::ImportFrom {
module,
names,
level,
},
line,
));
Ok(())
}
fn parse_exprlist(&mut self) -> Result<Expr, String> {
let first = self.parse_star_or_expr()?;
if self.at_op(",") {
let mut items = vec![first];
while self.eat_op(",") {
if self.stop_exprlist() {
break;
}
items.push(self.parse_star_or_expr()?);
}
Ok(Expr::Tuple(items))
} else {
Ok(first)
}
}
fn stop_exprlist(&self) -> bool {
self.at_newline()
|| matches!(self.cur(), Tok::Eof)
|| self.at_op("=")
|| self.at_op(";")
|| self.at_op(":")
|| self.at_op(")")
|| self.at_op("]")
|| self.at_op("}")
}
fn parse_star_or_expr(&mut self) -> Result<Expr, String> {
if self.eat_op("*") {
return Ok(Expr::Starred(Box::new(self.parse_expr()?)));
}
self.parse_namedexpr()
}
fn parse_namedexpr(&mut self) -> Result<Expr, String> {
let e = self.parse_ternary()?;
if self.at_op(":=") {
self.advance();
let v = self.parse_ternary()?;
return Ok(Expr::NamedExpr(Box::new(e), Box::new(v)));
}
Ok(e)
}
fn parse_expr(&mut self) -> Result<Expr, String> {
self.parse_namedexpr()
}
fn parse_ternary(&mut self) -> Result<Expr, String> {
let saved = self.depth;
self.enter()?;
if self.at_kw("lambda") {
let e = self.parse_lambda()?;
self.depth = saved;
return Ok(e);
}
let body_start = self.pos;
let body = self.parse_or()?;
if self.at_kw("if") {
self.advance();
let test = self.parse_or()?;
if !self.eat_kw("else") {
return Err(self.err_span(
"expected 'else' after 'if' expression",
body_start,
self.pos - 1,
));
}
let orelse = self.parse_ternary()?;
self.depth = saved;
return Ok(Expr::IfExp {
test: Box::new(test),
body: Box::new(body),
orelse: Box::new(orelse),
});
}
self.depth = saved;
Ok(body)
}
fn parse_lambda(&mut self) -> Result<Expr, String> {
self.advance(); let params = self.parse_params(":")?;
self.expect_op(":")?;
if self.at_kw("yield") {
return Err(self.err_here("invalid syntax"));
}
let body = self.parse_ternary()?;
Ok(Expr::Lambda {
params,
body: Box::new(body),
})
}
fn parse_or(&mut self) -> Result<Expr, String> {
let mut e = self.parse_and()?;
if self.at_kw("or") {
let mut items = vec![e];
while self.eat_kw("or") {
items.push(self.parse_and()?);
}
e = Expr::BoolOp(BoolOp::Or, items);
}
Ok(e)
}
fn parse_and(&mut self) -> Result<Expr, String> {
let mut e = self.parse_not()?;
if self.at_kw("and") {
let mut items = vec![e];
while self.eat_kw("and") {
items.push(self.parse_not()?);
}
e = Expr::BoolOp(BoolOp::And, items);
}
Ok(e)
}
fn parse_not(&mut self) -> Result<Expr, String> {
if self.eat_kw("not") {
let saved = self.depth;
self.enter()?;
let e = self.parse_not()?;
self.depth = saved;
return Ok(Expr::UnaryOp(UnOp::Not, Box::new(e)));
}
self.parse_comparison()
}
fn parse_comparison(&mut self) -> Result<Expr, String> {
let saved = self.depth;
let (sl, sc) = (self.line(), self.col());
let left = self.parse_bitor()?;
let mut ops = Vec::new();
loop {
self.enter()?;
let op = if self.at_op("<") {
CmpOp::Lt
} else if self.at_op(">") {
CmpOp::Gt
} else if self.at_op("<=") {
CmpOp::Le
} else if self.at_op(">=") {
CmpOp::Ge
} else if self.at_op("==") {
CmpOp::Eq
} else if self.at_op("!=") {
CmpOp::Ne
} else if self.at_kw("in") {
CmpOp::In
} else if self.at_kw("is") {
self.advance();
if self.eat_kw("not") {
ops.push((CmpOp::IsNot, self.parse_bitor()?));
} else {
ops.push((CmpOp::Is, self.parse_bitor()?));
}
continue;
} else if self.at_kw("not") {
self.advance();
if self.eat_kw("in") {
ops.push((CmpOp::NotIn, self.parse_bitor()?));
continue;
} else {
return Err(format!(
"SyntaxError: expected 'in' after 'not' (line {})",
self.line()
));
}
} else {
break;
};
self.advance();
ops.push((op, self.parse_bitor()?));
}
self.depth = saved;
if ops.is_empty() {
Ok(left)
} else {
let end = self.prev_end_col();
Ok(spanned(
Expr::Compare(Box::new(left), ops),
sl,
sc,
end,
0,
0,
))
}
}
fn parse_bitor(&mut self) -> Result<Expr, String> {
let saved = self.depth;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_bitxor()?;
while self.at_op("|") {
self.enter()?;
let (opc, ope) = (self.col(), self.cur_end_col());
self.advance();
let e2 = Expr::BinOp(BinOp::BitOr, Box::new(e), Box::new(self.parse_bitxor()?));
e = spanned(e2, sl, sc, self.prev_end_col(), opc, ope);
}
self.depth = saved;
Ok(e)
}
fn parse_bitxor(&mut self) -> Result<Expr, String> {
let saved = self.depth;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_bitand()?;
while self.at_op("^") {
self.enter()?;
let (opc, ope) = (self.col(), self.cur_end_col());
self.advance();
let e2 = Expr::BinOp(BinOp::BitXor, Box::new(e), Box::new(self.parse_bitand()?));
e = spanned(e2, sl, sc, self.prev_end_col(), opc, ope);
}
self.depth = saved;
Ok(e)
}
fn parse_bitand(&mut self) -> Result<Expr, String> {
let saved = self.depth;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_shift()?;
while self.at_op("&") {
self.enter()?;
let (opc, ope) = (self.col(), self.cur_end_col());
self.advance();
let e2 = Expr::BinOp(BinOp::BitAnd, Box::new(e), Box::new(self.parse_shift()?));
e = spanned(e2, sl, sc, self.prev_end_col(), opc, ope);
}
self.depth = saved;
Ok(e)
}
fn parse_shift(&mut self) -> Result<Expr, String> {
let saved = self.depth;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_arith()?;
loop {
let op = if self.at_op("<<") {
BinOp::Shl
} else if self.at_op(">>") {
BinOp::Shr
} else {
break;
};
self.enter()?;
let (opc, ope) = (self.col(), self.cur_end_col());
self.advance();
let e2 = Expr::BinOp(op, Box::new(e), Box::new(self.parse_arith()?));
e = spanned(e2, sl, sc, self.prev_end_col(), opc, ope);
}
self.depth = saved;
Ok(e)
}
fn parse_arith(&mut self) -> Result<Expr, String> {
let saved = self.depth;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_term()?;
loop {
let op = if self.at_op("+") {
BinOp::Add
} else if self.at_op("-") {
BinOp::Sub
} else {
break;
};
self.enter()?;
let (opc, ope) = (self.col(), self.cur_end_col());
self.advance();
let e2 = Expr::BinOp(op, Box::new(e), Box::new(self.parse_term()?));
let end = self.prev_end_col();
e = spanned(e2, sl, sc, end, opc, ope);
}
self.depth = saved;
Ok(e)
}
fn parse_term(&mut self) -> Result<Expr, String> {
let saved = self.depth;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_unary()?;
loop {
let op = if self.at_op("*") {
BinOp::Mul
} else if self.at_op("/") {
BinOp::Div
} else if self.at_op("//") {
BinOp::FloorDiv
} else if self.at_op("%") {
BinOp::Mod
} else if self.at_op("@") {
BinOp::MatMul
} else {
break;
};
self.enter()?;
let (opc, ope) = (self.col(), self.cur_end_col());
self.advance();
let e2 = Expr::BinOp(op, Box::new(e), Box::new(self.parse_unary()?));
let end = self.prev_end_col();
e = spanned(e2, sl, sc, end, opc, ope);
}
self.depth = saved;
Ok(e)
}
fn parse_unary(&mut self) -> Result<Expr, String> {
let (sl, sc) = (self.line(), self.col());
let unary = |p: &mut Self, op: UnOp| -> Result<Expr, String> {
let saved = p.depth;
p.enter()?;
p.advance();
let operand = p.parse_unary()?;
p.depth = saved;
let end = p.prev_end_col();
Ok(spanned(
Expr::UnaryOp(op, Box::new(operand)),
sl,
sc,
end,
0,
0,
))
};
if self.at_op("-") {
return unary(self, UnOp::Neg);
}
if self.at_op("+") {
return unary(self, UnOp::Pos);
}
if self.at_op("~") {
return unary(self, UnOp::Invert);
}
self.parse_power()
}
fn parse_power(&mut self) -> Result<Expr, String> {
let (sl, sc) = (self.line(), self.col());
let base = self.parse_await_postfix()?;
if self.at_op("**") {
let (opc, ope) = (self.col(), self.cur_end_col());
self.advance();
let exp = self.parse_unary()?; let end = self.prev_end_col();
return Ok(spanned(
Expr::BinOp(BinOp::Pow, Box::new(base), Box::new(exp)),
sl,
sc,
end,
opc,
ope,
));
}
Ok(base)
}
fn parse_await_postfix(&mut self) -> Result<Expr, String> {
let saved = self.depth;
let start = self.pos;
if self.eat_kw("await") {
self.enter()?;
let e = self.parse_await_postfix()?;
self.depth = saved;
return Ok(self.span_suspension(Expr::Await(Box::new(e)), start));
}
let (start_line, start_col) = (self.line(), self.col());
let mut e = self.parse_atom()?;
loop {
if self.at_op("(") {
self.enter()?;
let paren_col = self.col();
e = self.parse_call(e)?;
let end = self.prev_end_col();
e = spanned(e, start_line, start_col, end, paren_col, end);
} else if self.at_op("[") {
self.enter()?;
let bracket_col = self.col();
self.advance();
let sub = self.parse_subscript()?;
self.expect_op("]")?;
let end = self.prev_end_col();
e = spanned(
Expr::Subscript(Box::new(e), Box::new(sub)),
start_line,
start_col,
end,
bracket_col,
end,
);
} else if self.at_op(".") {
self.enter()?;
self.advance();
let attr = self.expect_name()?;
let end = self.prev_end_col();
e = spanned(
Expr::Attribute(Box::new(e), attr),
start_line,
start_col,
end,
0,
0,
);
} else {
break;
}
}
self.depth = saved;
Ok(e)
}
fn parse_call(&mut self, func: Expr) -> Result<Expr, String> {
self.expect_op("(")?;
let mut args = Vec::new();
let mut keywords = Vec::new();
let mut order = ArgOrder::default();
while !self.at_op(")") {
if self.eat_op("*") {
order.star()?;
args.push(Expr::Starred(Box::new(self.parse_expr()?)));
} else if self.eat_op("**") {
order.kw_unpack = true;
keywords.push(Keyword {
name: None,
value: self.parse_expr()?,
});
} else if matches!(self.cur(), Tok::Name(n) if !is_keyword(n))
&& matches!(&self.toks[self.pos + 1].tok, Tok::Op(o) if o == "=")
{
order.keyword = true;
let kn = self.expect_name()?;
self.expect_op("=")?;
let start = self.pos;
let value = self.parse_expr()?;
self.comma_hint(start)?;
keywords.push(Keyword {
name: Some(kn),
value,
});
} else {
order.positional()?;
let start = self.pos;
let e = self.parse_namedexpr()?;
if self.at_comp_for() {
let comps = self.parse_comprehension_clauses()?;
let alone = args.is_empty() && keywords.is_empty() && self.at_op(")");
if !alone {
return Err(self.err_span(
"Generator expression must be parenthesized",
start,
self.pos - 1,
));
}
args.push(Expr::GenExp(Box::new(e), comps));
} else {
self.comma_hint(start)?;
args.push(e);
}
}
if !self.eat_op(",") {
break;
}
}
self.expect_op(")")?;
Ok(Expr::Call {
func: Box::new(func),
args,
keywords,
})
}
fn parse_subscript(&mut self) -> Result<Expr, String> {
let parse_one = |p: &mut Self| -> Result<Expr, String> {
let lo_start = p.pos;
let lo = if p.at_op(":") {
None
} else {
Some(Box::new(p.parse_expr()?))
};
if lo.is_some() {
p.comma_hint(lo_start)?;
}
if p.at_op(":") {
p.advance();
let hi = if p.at_op(":") || p.at_op("]") || p.at_op(",") {
None
} else {
Some(Box::new(p.parse_expr()?))
};
let step = if p.eat_op(":") {
if p.at_op("]") || p.at_op(",") {
None
} else {
Some(Box::new(p.parse_expr()?))
}
} else {
None
};
Ok(Expr::Slice { lo, hi, step })
} else {
Ok(*lo.unwrap())
}
};
let first = parse_one(self)?;
if self.at_op(",") {
let mut items = vec![first];
while self.eat_op(",") {
if self.at_op("]") {
break;
}
items.push(parse_one(self)?);
}
Ok(Expr::Tuple(items))
} else {
Ok(first)
}
}
fn parse_atom(&mut self) -> Result<Expr, String> {
let line = self.line();
match self.cur().clone() {
Tok::Int(n) => {
self.advance();
Ok(Expr::Int(n))
}
Tok::BigInt(s) => {
self.advance();
Ok(Expr::BigInt(s))
}
Tok::Float(f) => {
self.advance();
Ok(Expr::Float(f))
}
Tok::Complex(f) => {
self.advance();
Ok(Expr::Complex(f))
}
Tok::Str(_) | Tok::FString(_, _) | Tok::TString(_, _) | Tok::Bytes(_) => {
self.parse_string_group()
}
Tok::Name(n) => {
let (nl, nc, ne) = (self.line(), self.col(), self.cur_end_col());
self.advance();
match n.as_str() {
"None" => Ok(Expr::None),
"True" => Ok(Expr::True),
"False" => Ok(Expr::False),
"lambda" => {
self.pos -= 1;
self.parse_lambda()
}
"yield" => {
let start = self.pos - 1;
let e = if self.eat_kw("from") {
Expr::YieldFrom(Box::new(self.parse_expr()?))
} else if self.at_newline()
|| self.at_op(")")
|| self.at_op("=")
|| self.at_op(";")
|| matches!(self.cur(), Tok::Eof)
{
Expr::Yield(None)
} else {
Expr::Yield(Some(Box::new(self.parse_exprlist()?)))
};
Ok(self.span_suspension(e, start))
}
_ if is_keyword(&n) => {
let _ = line;
Err(self.err_span("invalid syntax", self.pos - 1, self.pos - 1))
}
_ => Ok(spanned(Expr::Name(n), nl, nc, ne, 0, 0)),
}
}
Tok::Op(o) => match o.as_str() {
"(" => self.parse_paren(),
"[" => self.parse_list(),
"{" => self.parse_brace(),
"..." => {
self.advance();
Ok(Expr::Ellipsis)
}
_ => Err(self.err_here("invalid syntax")),
},
_ => {
let _ = line;
Err(self.err_here("invalid syntax"))
}
}
}
fn parse_string_group(&mut self) -> Result<Expr, String> {
let mut parts: Vec<FStrPart> = Vec::new();
let mut any_f = false;
let mut any_t = false;
let mut any_plain = false;
let mut byte_acc: Option<Vec<u8>> = None;
loop {
match self.cur().clone() {
Tok::Str(s) => {
self.advance();
any_plain = true;
parts.push(FStrPart::Lit(s));
}
Tok::Bytes(b) => {
self.advance();
byte_acc.get_or_insert_with(Vec::new).extend(b);
}
Tok::FString(raw, is_raw) => {
self.advance();
any_f = true;
let mut sub = self.parse_fstring(&raw, is_raw)?;
parts.append(&mut sub);
}
Tok::TString(raw, is_raw) => {
self.advance();
any_t = true;
let mut sub = self.parse_fstring(&raw, is_raw)?;
parts.append(&mut sub);
}
_ => break,
}
}
if any_t && (any_f || any_plain || byte_acc.is_some()) {
return Err(
"SyntaxError: cannot mix t-string literals with string or bytes literals"
.to_string(),
);
}
if byte_acc.is_some() && (any_plain || any_f || any_t) {
return Err("SyntaxError: cannot mix bytes and nonbytes literals".to_string());
}
if let Some(b) = byte_acc {
return Ok(Expr::Bytes(b));
}
if any_t {
Ok(Expr::TString(parts))
} else if any_f {
Ok(Expr::FString(parts))
} else {
let mut s = String::new();
for p in parts {
if let FStrPart::Lit(l) = p {
s.push_str(&l);
}
}
Ok(Expr::Str(s))
}
}
fn parse_fstring(&self, raw: &str, is_raw: bool) -> Result<Vec<FStrPart>, String> {
let chars: Vec<char> = raw.chars().collect();
let mut parts = Vec::new();
let mut lit = String::new();
let mut i = 0;
while i < chars.len() {
let c = chars[i];
if c == '{' {
if chars.get(i + 1) == Some(&'{') {
lit.push('{');
i += 2;
continue;
}
if crate::lexer::ends_with_named_escape_lead(&lit, is_raw) {
lit.push('{');
i += 1;
while i < chars.len() && chars[i] != '}' {
lit.push(chars[i]);
i += 1;
}
if i < chars.len() {
lit.push('}');
i += 1;
}
continue;
}
if !lit.is_empty() {
let decoded = crate::lexer::decode_escapes(&lit, is_raw)?;
parts.push(FStrPart::Lit(decoded));
lit.clear();
}
let mut depth = 1;
i += 1;
let mut field = String::new();
while i < chars.len() && depth > 0 {
match chars[i] {
'{' => {
depth += 1;
field.push('{');
}
'}' => {
depth -= 1;
if depth > 0 {
field.push('}');
}
}
other => field.push(other),
}
i += 1;
}
parts.extend(self.build_fstring_field(&field, is_raw)?);
} else if c == '}' {
if chars.get(i + 1) == Some(&'}') {
lit.push('}');
i += 2;
continue;
}
lit.push('}');
i += 1;
} else {
lit.push(c);
i += 1;
}
}
if !lit.is_empty() {
let decoded = crate::lexer::decode_escapes(&lit, is_raw)?;
parts.push(FStrPart::Lit(decoded));
}
Ok(parts)
}
fn build_fstring_field(&self, field: &str, is_raw: bool) -> Result<Vec<FStrPart>, String> {
let mut debug_prefix: Option<String> = None;
let work: String = if let Some(eq) = find_debug_eq(field) {
let after = &field[eq + 1..];
let ws = after.len() - after.trim_start().len();
debug_prefix = Some(field[..eq + 1 + ws].to_string());
format!("{}{}", &field[..eq], after.trim_start())
} else {
field.to_string()
};
let field: &str = &work;
let mut expr_src = field;
let mut spec: Vec<FStrPart> = Vec::new();
let mut conv: Option<char> = None;
if let Some(idx) = find_top_level(field, ':') {
spec = self.parse_fstring(&field[idx + 1..], is_raw)?;
expr_src = &field[..idx];
}
if expr_src.len() >= 2 {
let bytes = expr_src.as_bytes();
if bytes[expr_src.len() - 2] == b'!' {
let c = bytes[expr_src.len() - 1] as char;
if matches!(c, 's' | 'r' | 'a') {
conv = Some(c);
expr_src = &expr_src[..expr_src.len() - 2];
}
}
}
if debug_prefix.is_some() && conv.is_none() && spec.is_empty() {
conv = Some('r');
}
let expr_src = expr_src.trim();
let sub = parse(&format!("({expr_src})")).map_err(|e| format!("f-string: {e}"))?;
let expr = match sub.into_iter().next() {
Some(Stmt {
kind: StmtKind::Expr(e),
..
}) => e,
_ => return Err(format!("f-string: invalid expression {{{expr_src}}}")),
};
let mut out = Vec::with_capacity(2);
if let Some(pre) = debug_prefix {
out.push(FStrPart::Lit(pre));
}
out.push(FStrPart::Expr {
expr: Box::new(expr),
src: expr_src.to_string(),
conv,
spec,
});
Ok(out)
}
fn parse_paren(&mut self) -> Result<Expr, String> {
self.advance(); if self.eat_op(")") {
return Ok(Expr::Tuple(Vec::new()));
}
let start = self.pos;
let first = self.parse_star_or_expr()?;
if self.at_comp_for() {
let comps = self.parse_comprehension_clauses()?;
self.expect_op(")")?;
return Ok(Expr::GenExp(Box::new(first), comps));
}
self.comma_hint(start)?;
if self.at_op(",") {
let mut items = vec![first];
while self.eat_op(",") {
if self.at_op(")") {
break;
}
let start = self.pos;
items.push(self.parse_star_or_expr()?);
self.comma_hint(start)?;
}
self.expect_op(")")?;
return Ok(Expr::Tuple(items));
}
self.expect_op(")")?;
Ok(first)
}
fn parse_list(&mut self) -> Result<Expr, String> {
self.advance(); if self.eat_op("]") {
return Ok(Expr::List(Vec::new()));
}
let start = self.pos;
let first = self.parse_star_or_expr()?;
if self.at_comp_for() {
let comps = self.parse_comprehension_clauses()?;
self.expect_op("]")?;
return Ok(Expr::ListComp(Box::new(first), comps));
}
self.comma_hint(start)?;
let mut items = vec![first];
while self.eat_op(",") {
if self.at_op("]") {
break;
}
let start = self.pos;
items.push(self.parse_star_or_expr()?);
self.comma_hint(start)?;
}
self.expect_op("]")?;
Ok(Expr::List(items))
}
fn parse_brace(&mut self) -> Result<Expr, String> {
let (line, start) = (self.line(), self.col());
let e = self.parse_brace_display()?;
Ok(match e {
Expr::Dict(_) | Expr::Set(_) => spanned(e, line, start, self.prev_end_col(), 0, 0),
other => other,
})
}
fn parse_brace_display(&mut self) -> Result<Expr, String> {
self.advance(); if self.eat_op("}") {
return Ok(Expr::Dict(Vec::new()));
}
if self.eat_op("**") {
let v = self.parse_expr()?;
let mut pairs = vec![(None, v)];
while self.eat_op(",") {
if self.at_op("}") {
break;
}
if self.eat_op("**") {
pairs.push((None, self.parse_expr()?));
} else {
let k = self.parse_expr()?;
self.expect_op(":")?;
pairs.push((Some(k), self.parse_expr()?));
}
}
self.expect_op("}")?;
return Ok(Expr::Dict(pairs));
}
let first_start = self.pos;
let first = self.parse_star_or_expr()?;
if self.at_op(":") {
self.advance();
let v = self.parse_expr()?;
if self.at_comp_for() {
let comps = self.parse_comprehension_clauses()?;
self.expect_op("}")?;
return Ok(Expr::DictComp(Box::new(first), Box::new(v), comps));
}
let mut pairs = vec![(Some(first), v)];
while self.eat_op(",") {
if self.at_op("}") {
break;
}
if self.eat_op("**") {
pairs.push((None, self.parse_expr()?));
continue;
}
let k_start = self.pos;
let k = self.parse_expr()?;
if !self.at_op(":") {
let t = &self.toks[k_start];
return Err(at_pos(
"SyntaxError: ':' expected after dictionary key",
t.line,
t.col as i64 + 1,
t.line,
0,
));
}
self.advance();
pairs.push((Some(k), self.parse_expr()?));
}
self.expect_op("}")?;
Ok(Expr::Dict(pairs))
} else if self.at_comp_for() {
let comps = self.parse_comprehension_clauses()?;
self.expect_op("}")?;
Ok(Expr::SetComp(Box::new(first), comps))
} else {
self.comma_hint(first_start)?;
let mut items = vec![first];
while self.eat_op(",") {
if self.at_op("}") {
break;
}
let start = self.pos;
items.push(self.parse_star_or_expr()?);
self.comma_hint(start)?;
}
self.expect_op("}")?;
Ok(Expr::Set(items))
}
}
fn at_comp_for(&self) -> bool {
self.at_kw("for") || self.at_kw("async")
}
fn parse_comprehension_clauses(&mut self) -> Result<Vec<Comprehension>, String> {
let mut comps = Vec::new();
while self.at_kw("for") || self.at_kw("async") {
let is_async = self.eat_kw("async");
self.advance(); let target = self.parse_target_tuple()?;
if !self.eat_kw("in") {
return Err(format!(
"SyntaxError: comprehension missing 'in' (line {})",
self.line()
));
}
let iter = self.parse_or()?;
let mut ifs = Vec::new();
while self.at_kw("if") {
self.advance();
ifs.push(self.parse_or()?);
}
comps.push(Comprehension {
target: Box::new(target),
iter: Box::new(iter),
ifs,
is_async,
});
}
Ok(comps)
}
}
fn find_top_level(s: &str, ch: char) -> Option<usize> {
let mut depth = 0i32;
for (i, c) in s.char_indices() {
match c {
'(' | '[' | '{' => depth += 1,
')' | ']' | '}' => depth -= 1,
_ if c == ch && depth == 0 => return Some(i),
_ => {}
}
}
None
}
fn find_debug_eq(field: &str) -> Option<usize> {
let bytes = field.as_bytes();
let mut depth = 0i32;
let mut quote: Option<u8> = None;
let mut i = 0;
while i < bytes.len() {
let c = bytes[i];
if let Some(q) = quote {
if c == b'\\' {
i += 2;
continue;
}
if c == q {
quote = None;
}
i += 1;
continue;
}
match c {
b'\'' | b'"' => quote = Some(c),
b'(' | b'[' | b'{' => depth += 1,
b')' | b']' | b'}' => depth -= 1,
b':' if depth == 0 => return None,
b'=' if depth == 0 => {
let next = bytes.get(i + 1).copied();
let prev = if i > 0 {
bytes.get(i - 1).copied()
} else {
None
};
let is_eqeq = next == Some(b'=');
let is_cmp = matches!(
prev,
Some(b'=') | Some(b'!') | Some(b'<') | Some(b'>') | Some(b':')
);
if !is_eqeq && !is_cmp {
return Some(i);
}
}
_ => {}
}
i += 1;
}
None
}
fn augassign_op(o: &str) -> Option<BinOp> {
Some(match o {
"+=" => BinOp::Add,
"-=" => BinOp::Sub,
"*=" => BinOp::Mul,
"/=" => BinOp::Div,
"//=" => BinOp::FloorDiv,
"%=" => BinOp::Mod,
"**=" => BinOp::Pow,
"&=" => BinOp::BitAnd,
"|=" => BinOp::BitOr,
"^=" => BinOp::BitXor,
"<<=" => BinOp::Shl,
">>=" => BinOp::Shr,
"@=" => BinOp::MatMul,
_ => return None,
})
}
fn unspan(e: &Expr) -> &Expr {
match e {
Expr::Spanned(inner, _) => unspan(inner),
other => other,
}
}
fn expr_name(e: &Expr) -> &'static str {
match unspan(e) {
Expr::None => "None",
Expr::True => "True",
Expr::False => "False",
Expr::Ellipsis => "ellipsis",
Expr::Int(_)
| Expr::BigInt(_)
| Expr::Float(_)
| Expr::Complex(_)
| Expr::Str(_)
| Expr::Bytes(_) => "literal",
Expr::FString(_) => "f-string expression",
Expr::TString(_) => "t-string expression",
Expr::Name(_) => "name",
Expr::List(_) => "list",
Expr::Tuple(_) => "tuple",
Expr::Set(_) => "set display",
Expr::Dict(_) => "dict literal",
Expr::Starred(_) => "starred",
Expr::Compare(..) => "comparison",
Expr::IfExp { .. } => "conditional expression",
Expr::Call { .. } => "function call",
Expr::Attribute(..) => "attribute",
Expr::Subscript(..) => "subscript",
Expr::Slice { .. } => "slice",
Expr::Lambda { .. } => "lambda",
Expr::ListComp(..) => "list comprehension",
Expr::SetComp(..) => "set comprehension",
Expr::DictComp(..) => "dict comprehension",
Expr::GenExp(..) => "generator expression",
Expr::Yield(_) | Expr::YieldFrom(_) => "yield expression",
Expr::Await(_) => "await expression",
Expr::NamedExpr(..) => "named expression",
_ => "expression",
}
}