use crate::ast::*;
use crate::lexer::{lex, Tok, Token};
pub(crate) const KEYWORDS: &[&str] = &[
"False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue",
"def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import",
"in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while",
"with", "yield",
];
pub(crate) fn is_keyword(s: &str) -> bool {
KEYWORDS.contains(&s)
}
pub(crate) fn is_soft_keyword(s: &str) -> bool {
matches!(s, "match" | "case" | "type" | "_")
}
pub fn parse(src: &str) -> Result<Vec<Stmt>, String> {
crate::stack::with_frontend_stack(|| parse_source(src, 0))
.map_err(|e| with_metadata(e, &exec_input_source(src)))
}
fn parse_source(src: &str, level: u32) -> Result<Vec<Stmt>, String> {
let src = crate::rust_ffi::desugar(src);
let lexed = lex(&src)?;
let unclosed = lexed.unclosed;
let mut p = Parser::new(lexed.toks, lexed.deferred, &src);
p.level = level;
let err = match p.parse_module() {
Ok(stmts) => match unclosed {
Some(e) => e,
None => return Ok(stmts),
},
Err(e) => match unclosed {
Some(u) if p.at_end_of_input(true) => u,
_ => match p.deferred.take() {
Some(d) if matches!(p.cur(), Tok::Eof) => d,
_ => e,
},
},
};
let pos = split_syntax_error(&err).1;
let lineno = pos.as_ref().and_then(|p| p.lineno);
let multiline = pos.is_some_and(|p| p.end_lineno.zip(p.lineno).is_some_and(|(e, l)| e > l));
Err(match lineno.and_then(|l| source_line(&src, l, true)) {
Some(text) if multiline => with_text(err, text.trim_end_matches('\n')),
Some(text) => with_text(err, &text),
None => err,
})
}
pub const SYNTAX_FIELD: char = '\u{1}';
#[derive(Debug, Clone, Default, PartialEq)]
pub struct SyntaxPos {
pub lineno: Option<i64>,
pub offset: Option<i64>,
pub end_lineno: Option<i64>,
pub end_offset: Option<i64>,
pub text: Option<String>,
pub filename: Option<String>,
pub no_source: bool,
pub bare_args: bool,
pub metadata: Option<SyntaxMetadata>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct SyntaxMetadata {
pub lineno: i64,
pub offset: i64,
pub source: Option<String>,
}
pub fn with_metadata(err: String, source: &str) -> String {
let fields = err.find(SYNTAX_FIELD).map_or("", |i| &err[i..]);
if !fields.contains("\u{1}pos=")
|| fields.contains("\u{1}nosrc=")
|| fields.contains("\u{1}meta=")
{
return err;
}
let escaped = source.replace('\u{2}', "\u{2}2").replace(SYNTAX_FIELD, "\u{2}1");
format!("{err}{SYNTAX_FIELD}meta={escaped}")
}
fn unescape_metadata(value: &str) -> String {
let mut out = String::with_capacity(value.len());
let mut chars = value.chars();
while let Some(c) = chars.next() {
if c != '\u{2}' {
out.push(c);
continue;
}
match chars.next() {
Some('1') => out.push(SYNTAX_FIELD),
_ => out.push('\u{2}'),
}
}
out
}
pub fn exec_input_source(src: &str) -> String {
let mut s = translate_newlines(src);
if !s.ends_with('\n') {
s.push('\n');
}
s
}
pub fn translate_newlines(src: &str) -> String {
src.replace("\r\n", "\n").replace('\r', "\n")
}
pub fn at_pos(msg: &str, lineno: u32, offset: i64, end_lineno: u32, end_offset: i64) -> String {
format!("{msg}{SYNTAX_FIELD}pos={lineno}:{offset}:{end_lineno}:{end_offset}")
}
pub const CHAR_COLUMNS: &str = "\u{1}charcols=1";
pub fn byte_column(text: &str, col: u32) -> u32 {
let mut chars = 0;
let mut bytes = 0;
for c in text.chars().take(col as usize) {
chars += 1;
bytes += c.len_utf8() as u32;
}
bytes + (col - chars)
}
pub fn with_byte_columns(err: String, src: &str) -> String {
let Some(at) = err.find(CHAR_COLUMNS) else {
return err;
};
let mut err = err;
err.replace_range(at..at + CHAR_COLUMNS.len(), "");
let Some(field) = err.find("\u{1}pos=") else {
return err;
};
let value_start = field + "\u{1}pos=".len();
let value_end = err[value_start..].find(SYNTAX_FIELD).map_or(err.len(), |i| value_start + i);
let n: Vec<Option<i64>> = err[value_start..value_end].split(':').map(|s| s.parse().ok()).collect();
let [Some(l), Some(o), Some(el), Some(eo)] = n[..] else {
return err;
};
let lines: Vec<&str> = src.split('\n').collect();
let to_bytes = |line: i64, offset: i64| -> i64 {
let text = usize::try_from(line - 1).ok().and_then(|i| lines.get(i));
match text {
Some(text) if offset > 0 => byte_column(text.trim_end_matches('\r'), offset as u32 - 1) as i64 + 1,
_ => offset,
}
};
let value = format!("{l}:{}:{el}:{}", to_bytes(l, o), to_bytes(el, eo));
err.replace_range(value_start..value_end, &value);
err
}
pub fn with_text(err: String, text: &str) -> String {
let fields = err.find(SYNTAX_FIELD).map_or("", |i| &err[i..]);
if !fields.contains("\u{1}pos=")
|| fields.contains("\u{1}text=")
|| fields.contains("\u{1}nosrc=")
{
return err;
}
format!("{err}{SYNTAX_FIELD}text={text}")
}
pub fn with_filename(err: String, filename: &str) -> String {
let fields = err.find(SYNTAX_FIELD).map_or("", |i| &err[i..]);
if !fields.contains("\u{1}pos=") || fields.contains("\u{1}file=") {
return err;
}
format!("{err}{SYNTAX_FIELD}file={filename}")
}
pub fn split_syntax_error(err: &str) -> (&str, Option<SyntaxPos>) {
let Some(i) = err.find(SYNTAX_FIELD) else {
return (err, None);
};
let mut pos = SyntaxPos::default();
for field in err[i + 1..].split(SYNTAX_FIELD) {
let Some((key, value)) = field.split_once('=') else {
continue;
};
match key {
"pos" => {
let n: Vec<Option<i64>> = value.split(':').map(|s| s.parse().ok()).collect();
if let [l, o, el, eo] = n[..] {
pos.lineno = l;
pos.offset = o;
pos.end_lineno = el;
pos.end_offset = eo;
}
}
"text" => pos.text = Some(value.to_string()),
"nosrc" => pos.no_source = true,
"bare" => pos.bare_args = true,
"file" => pos.filename = Some(value.to_string()),
"meta" => {
pos.metadata = Some(SyntaxMetadata {
lineno: 0,
offset: 0,
source: Some(unescape_metadata(value)),
})
}
_ => {}
}
}
(&err[..i], Some(pos))
}
pub fn check_eval_source(given: &str) -> Result<(), String> {
let src = given.trim();
parse(src)
.map_err(|e| for_eval_input(e, src))
.and_then(|_| check_eval_input(src))
.map_err(|e| with_metadata(e, &translate_newlines(given.trim_start_matches([' ', '\t']))))
}
pub fn for_eval_input(err: String, src: &str) -> String {
let (head, Some(mut pos)) = split_syntax_error(&err) else {
return err;
};
let last_line = src.split_inclusive('\n').count() as i64;
let ends_without_newline = !src.ends_with('\n');
if let (Some(text), Some(l)) = (&pos.text, pos.lineno) {
if l == last_line && ends_without_newline {
let bare = text.trim_end_matches('\n').to_string();
let at_end = pos.offset == Some(bare.chars().count() as i64 + 1);
if at_end {
pos.offset = Some(0);
pos.end_offset = Some(0);
}
pos.text = Some(bare);
}
}
let mut out = head.to_string();
if let (Some(l), Some(o), Some(el), Some(eo)) =
(pos.lineno, pos.offset, pos.end_lineno, pos.end_offset)
{
out.push_str(&format!("{SYNTAX_FIELD}pos={l}:{o}:{el}:{eo}"));
}
if let Some(t) = &pos.text {
out.push_str(&format!("{SYNTAX_FIELD}text={t}"));
}
if let Some(f) = &pos.filename {
out.push_str(&format!("{SYNTAX_FIELD}file={f}"));
}
out
}
pub fn source_line(src: &str, lineno: i64, keep_newline: bool) -> Option<String> {
let idx = usize::try_from(lineno).ok()?.checked_sub(1)?;
let line = src.split_inclusive('\n').nth(idx)?;
Some(if keep_newline && !line.ends_with('\n') {
format!("{line}\n")
} else {
line.to_string()
})
}
pub fn render_syntax_error(pos: &SyntaxPos, msg: &str, default_file: &str) -> (String, String) {
let file = pos.filename.as_deref().unwrap_or(default_file);
let mut out = match pos.lineno {
Some(l) => format!(" File \"{file}\", line {l}\n"),
None => format!(" File \"{file}\"\n"),
};
if pos.text.is_none() {
return (out, msg.to_string());
}
let (pos, msg) = match crate::suggest::keyword_typo(msg, pos) {
Some((typo_pos, typo_msg)) => (typo_pos, typo_msg),
None => (pos.clone(), msg.to_string()),
};
out.push_str(&render_source_and_carets(&pos));
(out, msg)
}
pub fn render_syntax_head(pos: &SyntaxPos, head: &str, default_file: &str) -> String {
match head.split_once(": ") {
Some((class, msg)) => {
let (block, msg) = render_syntax_error(pos, msg, default_file);
format!("{block}{class}: {msg}")
}
None => format!("{}{head}", render_syntax_error(pos, "", default_file).0),
}
}
fn render_source_and_carets(pos: &SyntaxPos) -> String {
let mut out = String::new();
let Some(text) = &pos.text else {
return out;
};
let rtext = text.trim_end_matches('\n');
let ltext = rtext.trim_start_matches([' ', '\n', '\x0c']);
let spaces = (rtext.chars().count() - ltext.chars().count()) as i64;
out.push_str(&format!(" {ltext}\n"));
let Some(mut offset) = pos.offset else {
return out;
};
let len = rtext.chars().count() as i64;
let mut end = if pos.lineno == pos.end_lineno {
match pos.end_offset {
Some(e) if e != 0 => e,
_ => offset,
}
} else {
len + 1
};
let text_len = text.chars().count() as i64;
if offset > text_len {
offset = len + 1;
}
if end > text_len {
end = len + 1;
}
if offset >= end || end < 0 {
end = offset + 1;
}
let col = offset - 1 - spaces;
let end_col = end - 1 - spaces;
if col >= 0 {
let lead: String = ltext
.chars()
.take(col as usize)
.map(|c| if c.is_whitespace() { c } else { ' ' })
.collect();
out.push_str(&format!(
" {lead}{}\n",
"^".repeat((end_col - col).max(0) as usize)
));
}
out
}
pub fn check_eval_input(src: &str) -> Result<(), String> {
crate::stack::with_frontend_stack(|| check_eval_input_here(src))
}
fn check_eval_input_here(src: &str) -> Result<(), String> {
let lexed = lex(src)?;
let mut p = Parser::new(lexed.toks, None, src);
p.skip_newlines();
if matches!(p.cur(), Tok::Eof) {
return Err(with_text(
at_pos("SyntaxError: invalid syntax", 0, 0, 0, 0),
"",
));
}
let checked = p.parse_exprlist().and_then(|_| {
p.skip_newlines();
if matches!(p.cur(), Tok::Eof) {
Ok(())
} else {
Err(p.err_here("invalid syntax"))
}
});
let lineno = checked
.as_ref()
.err()
.and_then(|e| split_syntax_error(e).1?.lineno);
checked.map_err(|e| match lineno.and_then(|l| source_line(src, l, false)) {
Some(text) => with_text(e, &text),
None => e,
})
}
pub fn is_incomplete_input(src: &str) -> bool {
let src = crate::rust_ffi::desugar(src);
let lexed = match lex(&src) {
Ok(lexed) => lexed,
Err(e) => {
let (head, _) = split_syntax_error(&e);
return head.starts_with("SyntaxError: unterminated triple-quoted string literal")
|| head == "SyntaxError: unexpected EOF while parsing";
}
};
let in_bracket = lexed.unclosed.is_some();
let mut p = Parser::new(lexed.toks, lexed.deferred, &src);
if p.parse_module().is_ok() {
return in_bracket;
}
p.at_end_of_input(in_bracket)
}
const MAXSTACK: u32 = 6000;
const EXPRESSION_TO_LOOKAHEAD: u32 = 23;
const TOO_COMPLEX: &str =
"MemoryError: Parser stack overflowed - Python source too complex to parse";
#[derive(Default)]
struct ArgOrder {
keyword: bool,
kw_unpack: bool,
}
impl ArgOrder {
fn positional(&self) -> Result<(), String> {
if self.kw_unpack {
Err("SyntaxError: positional argument follows keyword argument unpacking".into())
} else if self.keyword {
Err("SyntaxError: positional argument follows keyword argument".into())
} else {
Ok(())
}
}
fn star(&self) -> Result<(), String> {
if self.kw_unpack {
Err(
"SyntaxError: iterable argument unpacking follows keyword argument unpacking"
.into(),
)
} else {
Ok(())
}
}
}
struct Parser {
toks: Vec<Token>,
pos: usize,
deferred: Option<String>,
level: u32,
lead: Vec<(usize, u32)>,
yield_at: Option<(usize, u32)>,
in_function: bool,
loop_depth: u32,
nesting: u32,
misplaced: Option<String>,
groups: std::collections::HashMap<usize, (usize, u32, u32)>,
lines: Vec<String>,
}
fn spanned(e: Expr, line: u32, start: u32, end: u32, anchor_start: u32, anchor_end: u32) -> Expr {
Expr::Spanned(
Box::new(e),
Span {
line,
end_line: line,
start,
end,
anchor_start,
anchor_end,
suppress: false,
},
)
}
impl Parser {
fn new(toks: Vec<Token>, deferred: Option<String>, src: &str) -> Parser {
Parser {
toks,
pos: 0,
deferred,
level: 0,
lead: Vec::new(),
yield_at: None,
in_function: false,
loop_depth: 0,
nesting: 0,
misplaced: None,
groups: std::collections::HashMap::new(),
lines: src.lines().map(str::to_string).collect(),
}
}
fn at_end_of_input(&self, in_bracket: bool) -> bool {
let mut end = self.toks.len();
while end > 0
&& (matches!(self.toks[end - 1].tok, Tok::Dedent | Tok::Eof)
|| in_bracket && matches!(self.toks[end - 1].tok, Tok::Newline))
{
end -= 1;
}
self.pos >= end
}
fn rule(&mut self, n: u32) -> Result<u32, String> {
let saved = self.level;
self.level += n;
self.reach(self.level)?;
Ok(saved)
}
fn reach(&self, level: u32) -> Result<(), String> {
if level > MAXSTACK || crate::stack::parser_exhausted() {
return Err(TOO_COMPLEX.to_string());
}
Ok(())
}
fn reach_absent_expression(&self, level: u32) -> Result<(), String> {
self.reach(level + EXPRESSION_TO_LOOKAHEAD)
}
fn under<T>(
&mut self,
level: u32,
f: impl FnOnce(&mut Self) -> Result<T, String>,
) -> Result<T, String> {
let saved = self.level;
self.level = level;
let r = f(self)?;
self.level = saved;
Ok(r)
}
fn cur(&self) -> &Tok {
&self.toks[self.pos].tok
}
fn line(&self) -> u32 {
self.toks[self.pos].line
}
fn col(&self) -> u32 {
self.toks[self.pos].col
}
fn cur_end_col(&self) -> u32 {
self.toks[self.pos].end_col
}
fn prev_end_col(&self) -> u32 {
self.toks[self.pos.saturating_sub(1)].end_col
}
fn node_start(&self, first: usize, end: usize) -> (u32, u32) {
match self.groups.get(&first) {
Some(&(close, line, col)) if close + 1 == end => (line, col),
_ => (self.toks[first].line, self.toks[first].col),
}
}
fn binop_tail(
&mut self,
left: Expr,
op: BinOp,
(sl, sc): (u32, u32),
parse_right: fn(&mut Self) -> Result<Expr, String>,
) -> Result<Expr, String> {
let op_tok = self.toks[self.pos].clone();
self.advance();
let right_first = self.pos;
let right = parse_right(self)?;
let mut anchor_end = op_tok.end_col;
let next = &self.toks[right_first];
if op_tok.end_col - op_tok.col == 1
&& next.line == op_tok.line
&& next.col == op_tok.end_col
&& self.node_start(right_first, self.pos) > (op_tok.line, op_tok.end_col)
{
anchor_end += 1;
}
let e = Expr::BinOp(op, Box::new(left), Box::new(right));
Ok(spanned(
e,
sl,
sc,
self.prev_end_col(),
op_tok.col,
anchor_end,
))
}
fn advance(&mut self) -> Tok {
let t = self.toks[self.pos].tok.clone();
if self.pos + 1 < self.toks.len() {
self.pos += 1;
}
t
}
fn at_op(&self, s: &str) -> bool {
matches!(self.cur(), Tok::Op(o) if o == s)
}
fn eat_op(&mut self, s: &str) -> bool {
if self.at_op(s) {
self.advance();
true
} else {
false
}
}
fn expect_op(&mut self, s: &str) -> Result<(), String> {
if self.eat_op(s) {
Ok(())
} else {
Err(self.err_here(if s == ":" && self.at_newline() {
"expected ':'"
} else {
"invalid syntax"
}))
}
}
fn at_kw(&self, kw: &str) -> bool {
matches!(self.cur(), Tok::Name(n) if n == kw)
}
fn eat_kw(&mut self, kw: &str) -> bool {
if self.at_kw(kw) {
self.advance();
true
} else {
false
}
}
fn at_newline(&self) -> bool {
matches!(self.cur(), Tok::Newline)
}
fn skip_newlines(&mut self) {
while matches!(self.cur(), Tok::Newline) {
self.advance();
}
}
fn at_identifier(&self) -> bool {
match self.cur() {
Tok::Name(n) => !is_keyword(n),
Tok::Ident(_) => true,
_ => false,
}
}
fn expect_name(&mut self) -> Result<String, String> {
match self.cur().clone() {
Tok::Name(n) if !is_keyword(&n) => {
self.advance();
Ok(n)
}
Tok::Ident(n) => {
self.advance();
Ok(n)
}
_ => Err(self.err_here("invalid syntax")),
}
}
fn err_here(&self, msg: &str) -> String {
if msg == "invalid syntax" {
match self.cur() {
Tok::Dedent => return self.unexpected_unindent(),
Tok::Indent => return self.unexpected_indent(),
_ => {}
}
}
self.err_span(msg, self.pos, self.pos)
}
fn unexpected_indent(&self) -> String {
let first = &self.toks[(self.pos + 1).min(self.toks.len() - 1)];
at_pos(
"IndentationError: unexpected indent",
first.line,
first.col as i64,
first.line,
-1,
)
}
fn unexpected_unindent(&self) -> String {
let next = self.toks[self.pos..]
.iter()
.find(|t| !matches!(t.tok, Tok::Dedent))
.filter(|t| !matches!(t.tok, Tok::Eof));
let (line, offset) = match next {
Some(t) => (t.line, t.col as i64),
None => {
if let Some(d) = &self.deferred {
return d.clone();
}
let nl = self.toks[..self.pos]
.iter()
.rev()
.find(|t| !matches!(t.tok, Tok::Dedent | Tok::Indent));
nl.map_or((self.line(), 0), |t| (t.line, t.col as i64 + 1))
}
};
at_pos(
"IndentationError: unexpected unindent",
line,
offset,
line,
-1,
)
}
fn err_span(&self, msg: &str, from: usize, to: usize) -> String {
self.err_span_as("SyntaxError", msg, from, to)
}
fn spanned_stmt(&self, kind: StmtKind, line: u32, start: usize) -> Stmt {
let mut stmt = Stmt::new(kind, line);
stmt.span = Some(self.token_span(start, self.pos - 1));
stmt
}
fn compiler_err_span(&self, msg: &str, from: usize, to: usize) -> String {
let (line, offset, end_line, end_offset) = self.token_span(from, to);
at_pos(
&format!("SyntaxError: {msg}"),
line,
self.byte_col(line, offset - 1) as i64 + 1,
end_line,
self.byte_col(end_line, end_offset - 1) as i64 + 1,
)
}
fn err_span_as(&self, class: &str, msg: &str, from: usize, to: usize) -> String {
let (line, offset, end_line, end_offset) = self.token_span(from, to);
at_pos(
&format!("{class}: {msg}"),
line,
offset as i64,
end_line,
end_offset as i64,
)
}
fn token_span(&self, from: usize, to: usize) -> (u32, u32, u32, u32) {
let a = &self.toks[from.min(self.toks.len() - 1)];
let b = &self.toks[to.min(self.toks.len() - 1)];
let end = match b.tok {
Tok::Newline | Tok::Eof | Tok::Indent | Tok::Dedent => b.col + 1,
_ => b.end_col,
};
(a.line, a.col + 1, b.line, end + 1)
}
fn span_suspension(&self, e: Expr, start: usize) -> Expr {
self.span_lines_from(e, start)
}
fn span_lines_from(&self, e: Expr, start: usize) -> Expr {
let (first, last) = (&self.toks[start], &self.toks[self.pos.saturating_sub(1)]);
let span = Span {
line: first.line,
end_line: last.line,
start: first.col,
end: last.end_col,
..Span::NONE
};
Expr::Spanned(Box::new(e), span)
}
fn span_from(&self, e: Expr, start: usize) -> Expr {
let (first, last) = (&self.toks[start], &self.toks[self.pos.saturating_sub(1)]);
if first.line != last.line {
return e;
}
spanned(e, first.line, first.col, last.end_col, 0, 0)
}
fn check_stmt_end(&self, stmt_start: usize) -> Result<(), String> {
if matches!(self.cur(), Tok::Newline | Tok::Eof | Tok::Dedent) || self.at_op(";") {
return Ok(());
}
let legacy = match &self.toks[stmt_start].tok {
Tok::Name(n) if self.pos == stmt_start + 1 && (n == "print" || n == "exec") => {
Some(n.clone())
}
_ => None,
};
if let (Some(n), true) = (legacy, self.starts_expression()) {
let mut last = self.pos;
while last + 1 < self.toks.len()
&& !matches!(self.toks[last + 1].tok, Tok::Newline | Tok::Eof)
{
last += 1;
}
return Err(self.err_span(
&format!("Missing parentheses in call to '{n}'. Did you mean {n}(...)?"),
stmt_start,
last,
));
}
Err(self.err_here("invalid syntax"))
}
fn comma_hint(&mut self, a_start: usize) -> Result<(), String> {
if !self.starts_expression() {
return Ok(());
}
let a = self.toks[a_start].tok.clone();
let single_name = self.pos == a_start + 1 && matches!(a, Tok::Name(_));
if single_name && matches!(self.cur(), Tok::Str(..)) {
return Ok(());
}
let legacy = match &a {
Tok::Name(n) if single_name && (n == "print" || n == "exec") => Some(n.clone()),
_ => None,
};
let b_start = self.pos;
let b_end = match self.parse_expr() {
Ok(_) => self.pos - 1,
Err(_) => b_start,
};
let msg = match legacy {
Some(n) => format!("Missing parentheses in call to '{n}'. Did you mean {n}(...)?"),
None => "invalid syntax. Perhaps you forgot a comma?".to_string(),
};
Err(self.err_span(&msg, a_start, b_end))
}
fn check_assign_targets(
&self,
targets: &[Expr],
spans: &[(usize, usize)],
) -> Result<(), String> {
let single = targets.len() == 1;
if let (true, Some(Expr::Tuple(items))) = (single, targets.first().map(unspan)) {
let (a, b) = spans[0];
let bare = !matches!(&self.toks[a].tok, Tok::Op(o) if o == "(");
if let (true, Some(last), Some(&(la, lb))) =
(bare, items.last(), self.split_commas(a, b).last())
{
let last = unspan(last);
let plain = !matches!(
last,
Expr::Name(_)
| Expr::Attribute(..)
| Expr::Subscript(..)
| Expr::Starred(_)
| Expr::Tuple(_)
| Expr::List(_)
| Expr::GenExp(..)
| Expr::True
| Expr::False
| Expr::None
| Expr::BoolOp(..)
| Expr::UnaryOp(UnOp::Not, _)
| Expr::Compare(..)
| Expr::IfExp { .. }
| Expr::Lambda { .. }
| Expr::NamedExpr(..)
);
if plain {
return Err(self.err_span(
&format!(
"cannot assign to {} here. Maybe you meant '==' instead of '='?",
expr_name(last)
),
la,
lb,
));
}
}
}
for (t, &(a, b)) in targets.iter().zip(spans) {
let Some((what, (ia, ib))) = self.invalid_target(t, a, b) else {
continue;
};
let e = unspan(t);
let comparable = !matches!(
e,
Expr::Tuple(_)
| Expr::List(_)
| Expr::GenExp(..)
| Expr::True
| Expr::False
| Expr::None
| Expr::BoolOp(..)
| Expr::UnaryOp(UnOp::Not, _)
| Expr::Compare(..)
| Expr::IfExp { .. }
| Expr::Lambda { .. }
| Expr::NamedExpr(..)
);
if matches!(e, Expr::Yield(_) | Expr::YieldFrom(_)) {
return Err(self.err_span("assignment to yield expression not possible", a, b));
}
let msg = if single && comparable {
format!("cannot assign to {what} here. Maybe you meant '==' instead of '='?")
} else {
format!("cannot assign to {what}")
};
return Err(self.err_span(&msg, ia, ib));
}
Ok(())
}
fn invalid_target(
&self,
e: &Expr,
a: usize,
b: usize,
) -> Option<(&'static str, (usize, usize))> {
self.invalid_target_of(e, a, b, false)
}
fn invalid_target_of(
&self,
e: &Expr,
a: usize,
b: usize,
for_targets: bool,
) -> Option<(&'static str, (usize, usize))> {
if matches!(self.groups.get(&a), Some(&(close, _, _)) if close == b) {
return self.invalid_target_of(e, a + 1, b - 1, for_targets);
}
match unspan(e) {
Expr::Name(_) | Expr::Attribute(..) | Expr::Subscript(..) => None,
Expr::Starred(inner) => self.invalid_target_of(inner, a + 1, b, for_targets),
Expr::Tuple(items) | Expr::List(items) => {
let (ia, ib) = self.strip_brackets(a, b);
let parts = self.split_commas(ia, ib);
items
.iter()
.zip(parts)
.find_map(|(item, (x, y))| self.invalid_target_of(item, x, y, for_targets))
}
Expr::Compare(left, ops) if for_targets => match ops.first() {
Some((CmpOp::In, _)) => {
let in_tok = self.top_level_in(a, b)?;
self.invalid_target_of(left, a, in_tok - 1, for_targets)
}
_ => None,
},
other => Some((expr_name(other), (a, b))),
}
}
fn star_error_span(&mut self, err: String) -> String {
let star = self.pos - 1;
let msg = err
.strip_prefix("SyntaxError: ")
.unwrap_or(&err)
.to_string();
let run_end = self.reparse(star, |p| loop {
p.expect_op("*")?;
p.parse_expr()?;
if !(p.at_op(",") && matches!(&p.toks[p.pos + 1].tok, Tok::Op(o) if o == "*")) {
return Ok(Expr::None);
}
p.advance();
});
let Some((_, last)) = run_end else {
return err;
};
let mut furthest = last + 1;
if matches!(&self.toks[furthest].tok, Tok::Op(o) if o == ",") {
furthest += 1;
}
let (line, offset, end_line, end_offset) = self.token_span(star - 1, furthest);
at_pos(
&format!("SyntaxError: {msg}"),
line,
offset as i64,
end_line,
end_offset as i64 - 1,
)
}
fn check_for_target(
&mut self,
target: &Expr,
start: usize,
comprehension: bool,
) -> Result<(), String> {
let invalid = self.invalid_target(target, start, self.pos - 1);
if self.at_kw("in") && invalid.is_none() {
return Ok(());
}
if comprehension {
let items = self.reparse(start, |p| {
p.parse_bitor()?;
while p.eat_op(",") && p.starts_expression() {
p.parse_bitor()?;
}
Ok(Expr::None)
});
if let Some((_, end)) = items {
if !matches!(&self.toks[end + 1].tok, Tok::Name(n) if n == "in") {
let msg = "'in' expected after for-loop variables";
return Err(self.err_span(msg, end + 1, end + 1));
}
}
}
if let Some(e) = self.invalid_for_target(start) {
return Err(e);
}
Err(match invalid {
Some((what, (a, b))) => self.err_span(&format!("cannot assign to {what}"), a, b),
None => self.err_here("invalid syntax"),
})
}
fn invalid_for_target(&mut self, start: usize) -> Option<String> {
let (target, end) = self.reparse(start, |p| p.parse_exprlist())?;
let (what, (a, b)) = self.invalid_target_of(&target, start, end, true)?;
Some(self.err_span(&format!("cannot assign to {what}"), a, b))
}
fn reparse(
&mut self,
start: usize,
parse: impl FnOnce(&mut Self) -> Result<Expr, String>,
) -> Option<(Expr, usize)> {
let saved = (self.pos, self.level, self.misplaced.clone());
self.pos = start;
let parsed = parse(self).ok().map(|e| (e, self.pos - 1));
(self.pos, self.level, self.misplaced) = saved;
parsed
}
fn top_level_in(&self, a: usize, b: usize) -> Option<usize> {
let mut depth = 0i32;
(a..=b).find(|&i| {
match &self.toks[i].tok {
Tok::Op(o) if matches!(o.as_str(), "(" | "[" | "{") => depth += 1,
Tok::Op(o) if matches!(o.as_str(), ")" | "]" | "}") => depth -= 1,
Tok::Name(n) => return depth == 0 && n == "in",
_ => {}
}
false
})
}
fn check_with_target(&self, target: &Expr, start: usize) -> Result<(), String> {
if !(self.at_op(",") || self.at_op(")") || self.at_op(":")) {
return Ok(());
}
match self.invalid_target(target, start, self.pos - 1) {
Some((what, (a, b))) => Err(self.err_span(&format!("cannot assign to {what}"), a, b)),
None => Ok(()),
}
}
fn strip_brackets(&self, a: usize, b: usize) -> (usize, usize) {
let open = matches!(&self.toks[a].tok, Tok::Op(o) if o == "(" || o == "[");
let close = matches!(&self.toks[b].tok, Tok::Op(o) if o == ")" || o == "]");
if open && close && b > a + 1 && self.split_commas(a, b).len() == 1 {
(a + 1, b - 1)
} else {
(a, b)
}
}
fn split_commas(&self, a: usize, b: usize) -> Vec<(usize, usize)> {
let mut out = Vec::new();
let mut depth = 0i32;
let mut start = a;
for i in a..=b.min(self.toks.len() - 1) {
match &self.toks[i].tok {
Tok::Op(o) if matches!(o.as_str(), "(" | "[" | "{") => depth += 1,
Tok::Op(o) if matches!(o.as_str(), ")" | "]" | "}") => depth -= 1,
Tok::Op(o) if o == "," && depth == 0 => {
if i > start {
out.push((start, i - 1));
}
start = i + 1;
}
_ => {}
}
}
if start <= b {
out.push((start, b));
}
out
}
fn starts_expression(&self) -> bool {
match self.cur() {
Tok::Name(n) => {
!is_keyword(n)
|| matches!(
n.as_str(),
"None" | "True" | "False" | "not" | "lambda" | "await"
)
}
Tok::Op(o) => matches!(o.as_str(), "(" | "[" | "{" | "-" | "+" | "~" | "..."),
Tok::Newline | Tok::Indent | Tok::Dedent | Tok::Eof => false,
_ => true,
}
}
fn parse_module(&mut self) -> Result<Vec<Stmt>, String> {
self.rule(3)?;
let mut stmts = Vec::new();
self.skip_newlines();
while !matches!(self.cur(), Tok::Eof) {
self.parse_statement(&mut stmts)?;
self.skip_newlines();
}
match self.deferred.take().or_else(|| self.misplaced.take()) {
Some(e) => Err(e),
None => Ok(stmts),
}
}
fn note_misplaced(&mut self, msg: &str, start: usize) {
if self.misplaced.is_none() {
let e = self.compiler_err_span(msg, start, self.pos.saturating_sub(1));
self.misplaced = Some(format!("{e}{SYNTAX_FIELD}nosrc=1"));
}
}
fn parse_loop_body(&mut self, desc: &str, line: u32) -> Result<Vec<Stmt>, String> {
self.loop_depth += 1;
let body = self.parse_suite(desc, line);
self.loop_depth -= 1;
body
}
fn parse_scope_body(
&mut self,
desc: &str,
line: u32,
function: bool,
) -> Result<Vec<Stmt>, String> {
let saved = (self.in_function, self.loop_depth);
self.in_function = function;
self.loop_depth = 0;
self.nesting += 1;
let body = self.parse_suite(desc, line);
self.nesting -= 1;
(self.in_function, self.loop_depth) = saved;
body
}
fn parse_suite(&mut self, desc: &str, kw_line: u32) -> Result<Vec<Stmt>, String> {
let forced = matches!(
desc,
"function definition" | "'try' statement" | "'else' statement" | "'finally' statement"
);
if forced && !self.at_op(":") {
return Err(self.err_here("expected ':'"));
}
self.expect_op(":")?;
if self.at_newline() {
self.skip_newlines();
if !matches!(self.cur(), Tok::Indent) {
return Err(self.err_span_as(
"IndentationError",
&format!("expected an indented block after {desc} on line {kw_line}"),
self.pos,
self.pos,
));
}
self.advance(); let statements = self.level + 3;
let mut body = Vec::new();
while !matches!(self.cur(), Tok::Dedent | Tok::Eof) {
self.under(statements, |p| p.parse_statement(&mut body))?;
self.skip_newlines();
}
if matches!(self.cur(), Tok::Dedent) {
self.advance();
}
Ok(body)
} else {
let mut body = Vec::new();
let block = self.level + 1;
self.under(block, |p| p.parse_simple_line(&mut body))?;
Ok(body)
}
}
fn parse_statement(&mut self, out: &mut Vec<Stmt>) -> Result<(), String> {
let saved = self.rule(1)?;
self.parse_statement_kind(out)?;
self.level = saved;
Ok(())
}
fn parse_statement_kind(&mut self, out: &mut Vec<Stmt>) -> Result<(), String> {
let line = self.line();
if matches!(self.cur(), Tok::Indent) {
return Err(self.unexpected_indent());
}
if let Tok::Name(n) = self.cur().clone() {
match n.as_str() {
"if" => return self.parse_if(out, line),
"while" => return self.parse_while(out, line),
"for" => return self.parse_for(out, line, false),
"def" => return self.parse_funcdef(out, line, Vec::new(), false),
"class" => return self.parse_classdef(out, line, Vec::new()),
"try" => return self.parse_try(out, line),
"with" => return self.parse_with(out, line, false),
"async" => return self.parse_async(out, line),
"match" if self.looks_like_match() => return self.parse_match(out, line),
_ => {}
}
}
if self.at_op("@") {
return self.parse_decorated(out, line);
}
if self.looks_like_stray_case() {
return Err("SyntaxError: invalid syntax. Did you mean 'class'?".to_string());
}
self.parse_simple_line(out)
}
fn looks_like_stray_case(&self) -> bool {
matches!(self.cur(), Tok::Name(n) if n == "case")
&& matches!(
self.toks.get(self.pos + 1).map(|t| &t.tok),
Some(Tok::Name(_))
)
&& matches!(
self.toks.get(self.pos + 2).map(|t| &t.tok),
Some(Tok::Op(o)) if o == ":"
)
}
fn parse_simple_line(&mut self, out: &mut Vec<Stmt>) -> Result<(), String> {
let saved = self.rule(1)?;
let simple_stmts = self.level;
let mut first = true;
loop {
let stmt_start = self.pos;
let level = if first {
simple_stmts
} else {
simple_stmts + 2
};
first = false;
self.under(level, |p| p.parse_simple_stmt(out))?;
self.check_stmt_end(stmt_start)?;
if self.eat_op(";") {
if self.at_newline() || matches!(self.cur(), Tok::Eof) {
break;
}
continue;
}
break;
}
if self.at_newline() {
self.advance();
}
self.level = saved;
Ok(())
}
fn parse_simple_stmt(&mut self, out: &mut Vec<Stmt>) -> Result<(), String> {
let saved = self.rule(1)?;
self.parse_simple_stmt_kind(out)?;
self.level = saved;
Ok(())
}
fn parse_simple_stmt_kind(&mut self, out: &mut Vec<Stmt>) -> Result<(), String> {
let line = self.line();
let stmt = self.level + 1;
if let Tok::Name(n) = self.cur().clone() {
match n.as_str() {
"pass" => {
self.advance();
out.push(Stmt::new(StmtKind::Pass, line));
return Ok(());
}
"break" => {
let start = self.pos;
self.advance();
if self.loop_depth == 0 {
self.note_misplaced("'break' outside loop", start);
}
out.push(self.spanned_stmt(StmtKind::Break, line, start));
return Ok(());
}
"continue" => {
let start = self.pos;
self.advance();
if self.loop_depth == 0 {
self.note_misplaced("'continue' not properly in loop", start);
}
out.push(self.spanned_stmt(StmtKind::Continue, line, start));
return Ok(());
}
"return" => {
let start = self.pos;
self.advance();
let v =
if self.at_newline() || self.at_op(";") || matches!(self.cur(), Tok::Eof) {
self.reach_absent_expression(stmt + 3)?;
None
} else {
Some(self.under(stmt, Self::parse_exprlist)?)
};
if !self.in_function {
self.note_misplaced("'return' outside function", start);
}
out.push(self.spanned_stmt(StmtKind::Return(v), line, start));
return Ok(());
}
"raise" => return self.parse_raise(out, line),
"del" => {
self.advance();
let start = self.pos;
let targets = self.parse_target_list(stmt + 5)?;
let items = self.split_commas(start, self.pos - 1);
for (t, (a, b)) in targets.iter().zip(items) {
if let Some((what, (a, b))) = self.invalid_target(t, a, b) {
return Err(self.err_span(&format!("cannot delete {what}"), a, b));
}
}
out.push(Stmt::new(StmtKind::Delete(targets), line));
return Ok(());
}
"assert" => {
self.advance();
let test = self.under(stmt, Self::parse_expr)?;
let msg = if self.eat_op(",") {
Some(self.under(stmt, Self::parse_expr)?)
} else {
None
};
out.push(Stmt::new(StmtKind::Assert { test, msg }, line));
return Ok(());
}
"global" => {
let start = self.pos;
self.advance();
let names = self.parse_name_list()?;
let mut stmt = Stmt::new(StmtKind::Global(names), line);
stmt.span = Some(self.token_span(start, self.pos - 1));
out.push(stmt);
return Ok(());
}
"nonlocal" => {
let start = self.pos;
self.advance();
let names = self.parse_name_list()?;
if self.nesting == 0 && self.misplaced.is_none() {
self.note_misplaced(
"nonlocal declaration not allowed at module level",
start,
);
if let Some(e) = &mut self.misplaced {
e.push_str(&format!("{SYNTAX_FIELD}bare=1"));
}
}
let mut stmt = Stmt::new(StmtKind::Nonlocal(names), line);
stmt.span = Some(self.token_span(start, self.pos - 1));
out.push(stmt);
return Ok(());
}
"import" => return self.parse_import(out, line),
"from" => return self.parse_from_import(out, line),
"type" if self.looks_like_type_alias() => return self.parse_type_alias(out, line),
_ => {}
}
}
self.parse_expr_stmt(out, line)
}
fn looks_like_type_alias(&self) -> bool {
let tok = |i: usize| self.toks.get(self.pos + i).map(|t| &t.tok);
let name = match tok(1) {
Some(Tok::Name(n)) => !is_keyword(n),
Some(Tok::Ident(_)) => true,
_ => false,
};
name && matches!(tok(2), Some(Tok::Op(o)) if o == "=" || o == "[")
}
fn parse_type_alias(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance(); let name = self.expect_name()?;
let params = self.parse_type_params()?;
self.expect_op("=")?;
let value = self.under(self.level + 1, Self::parse_expr)?;
out.push(Stmt::new(
StmtKind::TypeAlias {
name,
params,
value,
},
line,
));
Ok(())
}
fn parse_name_list(&mut self) -> Result<Vec<String>, String> {
let mut names = vec![self.expect_name()?];
while self.eat_op(",") {
names.push(self.expect_name()?);
}
Ok(names)
}
fn parse_target_list(&mut self, raw: u32) -> Result<Vec<Expr>, String> {
self.lead = vec![(self.pos, raw)];
let mut ts = vec![self.parse_expr()?];
while self.eat_op(",") {
if self.at_newline() || matches!(self.cur(), Tok::Eof) {
break;
}
self.lead = vec![(self.pos, raw + 1)];
ts.push(self.parse_expr()?);
}
Ok(ts)
}
fn parse_expr_stmt(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let simple_stmt = self.level;
let first_start = self.pos;
if self.at_kw("yield") {
self.yield_at = Some((self.pos, simple_stmt + 2));
}
self.lead = vec![(self.pos, simple_stmt + 5)];
if self.at_op("(") {
self.lead.push((self.pos + 1, simple_stmt + 6));
}
let first = self.parse_exprlist()?;
if matches!(first.unspanned(), Expr::NamedExpr(..))
&& matches!(self.toks[first_start].tok, Tok::Name(_))
{
return Err(self.err_span("invalid syntax", first_start + 1, first_start + 1));
}
if self.at_op(":") {
self.advance();
let annotation = self.under(simple_stmt + 1, Self::parse_expr)?;
let value = if self.eat_op("=") {
if self.at_kw("yield") {
self.yield_at = Some((self.pos, simple_stmt + 4));
}
Some(self.under(simple_stmt + 3, Self::parse_exprlist)?)
} else {
None
};
out.push(Stmt::new(
StmtKind::AnnAssign {
target: first,
annotation,
value,
},
line,
));
return Ok(());
}
let first_end = self.pos.saturating_sub(1);
if let Tok::Op(o) = self.cur().clone() {
if let Some(op) = augassign_op(&o) {
let target = unspan(&first);
if !matches!(
target,
Expr::Name(_) | Expr::Attribute(..) | Expr::Subscript(..)
) {
return Err(self.err_span(
&format!(
"'{}' is an illegal expression for augmented assignment",
expr_name(target)
),
first_start,
first_end,
));
}
self.advance();
if self.at_kw("yield") {
self.yield_at = Some((self.pos, simple_stmt + 3));
}
let value = self.under(simple_stmt + 2, Self::parse_exprlist)?;
out.push(Stmt::new(
StmtKind::AugAssign {
target: first,
op,
value,
},
line,
));
return Ok(());
}
}
if self.at_op("=") {
let mut targets = vec![first];
let mut spans = vec![(first_start, first_end)];
let mut value = None;
while self.eat_op("=") {
let start = self.pos;
if self.at_kw("yield") {
self.yield_at = Some((self.pos, simple_stmt + 3));
}
self.lead = vec![(self.pos, simple_stmt + 8)];
let e = self.under(simple_stmt + 2, Self::parse_exprlist)?;
if let Some(prev) = value.take() {
targets.push(prev);
}
spans.push((start, self.pos - 1));
value = Some(e);
}
self.check_assign_targets(&targets, &spans)?;
for (t, &(from, to)) in targets.iter_mut().zip(&spans) {
if matches!(t, Expr::Tuple(_) | Expr::List(_)) {
let (a, b) = (&self.toks[from], &self.toks[to]);
if a.line == b.line {
let bare = std::mem::replace(t, Expr::Tuple(Vec::new()));
*t = spanned(bare, a.line, a.col, b.end_col, 0, 0);
}
}
}
out.push(Stmt::new(
StmtKind::Assign {
targets,
value: value.unwrap(),
},
line,
));
return Ok(());
}
out.push(Stmt::new(StmtKind::Expr(first), line));
Ok(())
}
fn parse_if(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let saved = self.rule(2)?;
self.parse_if_clause(out, line)?;
self.level = saved;
Ok(())
}
fn parse_if_clause(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let clause = self.level;
let kw = if matches!(self.cur(), Tok::Name(n) if n == "elif") {
"'elif' statement"
} else {
"'if' statement"
};
self.advance(); let test = self.under(clause + 1, Self::parse_namedexpr)?;
let body = self.parse_suite(kw, line)?;
let mut orelse = Vec::new();
self.skip_newlines_shallow();
if self.at_kw("elif") {
let elif_line = self.line();
self.under(clause + 1, |p| p.parse_if_clause(&mut orelse, elif_line))?;
} else if self.at_kw("else") {
let else_line = self.line();
self.advance();
orelse = self.under(clause + 1, |p| p.parse_suite("'else' statement", else_line))?;
}
out.push(Stmt::new(StmtKind::If { test, body, orelse }, line));
Ok(())
}
fn skip_newlines_shallow(&mut self) {
}
fn parse_while(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let saved = self.rule(2)?;
let rule = self.level;
self.advance();
let test = self.under(rule + 1, Self::parse_namedexpr)?;
let body = self.parse_loop_body("'while' statement", line)?;
let orelse = if self.at_kw("else") {
let el = self.line();
self.advance();
self.under(rule + 1, |p| p.parse_suite("'else' statement", el))?
} else {
Vec::new()
};
self.level = saved;
out.push(Stmt::new(StmtKind::While { test, body, orelse }, line));
Ok(())
}
fn parse_for(&mut self, out: &mut Vec<Stmt>, line: u32, is_async: bool) -> Result<(), String> {
let saved = self.rule(2)?;
let rule = self.level;
self.advance();
let start = self.pos;
let target = self.parse_target_tuple()?;
self.check_for_target(&target, start, false)?;
self.advance(); let iter = self.parse_exprlist()?;
let body = self.parse_loop_body("'for' statement", line)?;
let orelse = if self.at_kw("else") {
let el = self.line();
self.advance();
self.under(rule + 1, |p| p.parse_suite("'else' statement", el))?
} else {
Vec::new()
};
self.level = saved;
out.push(Stmt::new(
StmtKind::For {
target,
iter,
body,
orelse,
is_async,
},
line,
));
Ok(())
}
fn parse_target_tuple(&mut self) -> Result<Expr, String> {
let start = self.pos;
let first = self.parse_target_atom()?;
let target = if self.at_op(",") {
let mut items = vec![first];
while self.eat_op(",") {
if self.at_kw("in") || self.at_op("=") || self.at_op(":") {
break;
}
items.push(self.parse_target_atom()?);
}
Expr::Tuple(items)
} else {
first
};
Ok(self.span_unpack_target(target, start))
}
fn span_unpack_target(&self, target: Expr, start: usize) -> Expr {
match target {
Expr::Tuple(_) | Expr::List(_) => self.span_from(target, start),
other => other,
}
}
fn parse_target_atom(&mut self) -> Result<Expr, String> {
if self.eat_op("*") {
return Ok(Expr::Starred(Box::new(self.parse_await_postfix()?)));
}
self.parse_await_postfix()
}
fn parenthesized_with_items(&mut self) -> Result<Option<Vec<WithItem>>, String> {
if !self.at_op("(") {
return Ok(None);
}
let save = (self.pos, self.level);
let with_stmt = self.level;
self.advance();
let mut items = Vec::new();
let shaped = loop {
if self.at_op(")") {
break !items.is_empty();
}
let item = with_stmt + if items.is_empty() { 2 } else { 3 };
let Ok(context) = self.under(item, Self::parse_expr) else {
break false;
};
let vars = if self.eat_kw("as") {
let start = self.pos;
let Ok(v) = self.parse_ternary() else {
break false;
};
self.check_with_target(&v, start)?;
Some(self.span_unpack_target(v, start))
} else {
None
};
items.push(WithItem { context, vars });
if !self.eat_op(",") {
break true;
}
};
if shaped && self.eat_op(")") && self.at_op(":") {
return Ok(Some(items));
}
(self.pos, self.level) = save;
Ok(None)
}
fn parse_with(&mut self, out: &mut Vec<Stmt>, line: u32, is_async: bool) -> Result<(), String> {
let saved = self.rule(2)?;
let with_stmt = self.level;
self.advance();
let items = match self.parenthesized_with_items()? {
Some(items) => items,
None => {
let mut items = Vec::new();
loop {
let item = with_stmt + if items.is_empty() { 2 } else { 3 };
let context = self.under(item, Self::parse_expr)?;
let vars = if self.eat_kw("as") {
let start = self.pos;
let v = self.parse_ternary()?;
self.check_with_target(&v, start)?;
Some(self.span_unpack_target(v, start))
} else {
None
};
items.push(WithItem { context, vars });
if !self.eat_op(",") {
break;
}
}
items
}
};
let body = self.parse_suite("'with' statement", line)?;
self.level = saved;
out.push(Stmt::new(
StmtKind::With {
items,
body,
is_async,
},
line,
));
Ok(())
}
fn parse_async(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance(); if self.at_kw("def") {
return self.parse_funcdef(out, line, Vec::new(), true);
}
if self.at_kw("for") {
return self.parse_for(out, line, true);
}
if self.at_kw("with") {
return self.parse_with(out, line, true);
}
let _ = line;
Err(self.err_here("invalid syntax"))
}
fn parse_decorated(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let decorator = self.level + 6;
let mut decorators = Vec::new();
while self.eat_op("@") {
decorators.push(self.under(decorator, Self::parse_namedexpr)?);
if self.at_newline() {
self.advance();
}
self.skip_newlines();
}
if self.eat_kw("async") {
return self.parse_funcdef(out, line, decorators, true);
}
if self.at_kw("def") {
self.parse_funcdef(out, line, decorators, false)
} else if self.at_kw("class") {
self.parse_classdef(out, line, decorators)
} else {
match self.cur() {
Tok::Dedent => return Err(self.unexpected_unindent()),
Tok::Indent => return Err(self.unexpected_indent()),
_ => {}
}
let _ = line;
Err("SyntaxError: invalid syntax".to_string())
}
}
fn parse_type_params(&mut self) -> Result<Vec<TypeParam>, String> {
let mut names = Vec::new();
if !self.at_op("[") {
return Ok(names);
}
let mut kind = TypeParamKind::TypeVar;
self.advance(); let mut depth = 1usize;
let mut expect_name = true;
loop {
match self.cur().clone() {
Tok::Op(o) if o == "[" => {
depth += 1;
expect_name = false;
}
Tok::Op(o) if o == "]" => {
depth -= 1;
if depth == 0 {
self.advance();
return Ok(names);
}
}
Tok::Op(o) if depth == 1 && o == "," => {
expect_name = true;
kind = TypeParamKind::TypeVar;
}
Tok::Op(o) if depth == 1 && expect_name && o == "*" => {
kind = TypeParamKind::TypeVarTuple;
}
Tok::Op(o) if depth == 1 && expect_name && o == "**" => {
kind = TypeParamKind::ParamSpec;
}
Tok::Op(_) if depth == 1 => expect_name = false, Tok::Name(name) | Tok::Ident(name) if depth == 1 && expect_name => {
names.push(TypeParam { name, kind });
expect_name = false;
}
Tok::Eof => return Err("SyntaxError: unterminated type-parameter list".to_string()),
_ => {}
}
self.advance();
}
}
fn bind_type_params(&self, out: &mut Vec<Stmt>, params: &[TypeParam], line: u32) {
for p in params {
out.push(Stmt::new(
StmtKind::Assign {
targets: vec![Expr::Name(p.name.clone())],
value: Expr::Name("object".to_string()),
},
line,
));
}
}
fn parse_funcdef(
&mut self,
out: &mut Vec<Stmt>,
line: u32,
decorators: Vec<Expr>,
is_async: bool,
) -> Result<(), String> {
let saved = self.rule(3)?;
let rule = self.level;
self.advance(); let name = self.expect_name()?;
let type_params = self.parse_type_params()?; self.bind_type_params(out, &type_params, line);
self.expect_op("(")?;
let mut params = self.parse_params(")")?;
self.expect_op(")")?;
if self.eat_op("->") {
let ret = self.under(rule + 1, Self::parse_expr)?;
params.annotations.push(("return".to_string(), ret));
}
let body = self.parse_scope_body("function definition", line, true)?;
self.level = saved;
out.push(Stmt::new(
StmtKind::FuncDef {
name,
params,
body,
decorators,
is_async,
},
line,
));
Ok(())
}
fn parse_params(&mut self, close: &str) -> Result<Params, String> {
let default_level = self.level + 6;
let annotation_level = self.level + 7;
let mut p = Params::default();
let mut seen_star = false;
let mut seen_default = false;
let mut seen_kwargs = false;
let mut names_seen: Vec<String> = Vec::new();
let mut check_dup = |parser: &mut Self, name: &str, at: usize| {
if names_seen.iter().any(|n| n == name) {
if parser.misplaced.is_none() {
let e = parser.compiler_err_span(
&format!("duplicate argument '{name}' in function definition"),
at,
at,
);
parser.misplaced =
Some(format!("{e}{SYNTAX_FIELD}nosrc=1{SYNTAX_FIELD}bare=1"));
}
} else {
names_seen.push(name.to_string());
}
};
loop {
if self.at_op(close) {
break;
}
if seen_kwargs {
return Err(self.err_here("arguments cannot follow var-keyword argument"));
}
if self.eat_op("/") {
p.posonly = p.names.len();
let _ = self.eat_op(",");
continue;
}
if self.at_op("*") {
if seen_star {
return Err(self.err_here("* argument may appear only once"));
}
let star_at = self.pos;
self.advance();
if self.at_op(close)
|| self.at_op(",")
&& matches!(&self.toks[self.pos + 1].tok, Tok::Op(o) if o == close || o == "**")
{
let at = if close == ")" { star_at } else { self.pos };
return Err(self.err_span("named arguments must follow bare *", at, at));
}
if self.at_op(",") {
p.star = Some(String::new()); } else {
let name_at = self.pos;
let star_name = self.expect_name()?;
check_dup(self, &star_name, name_at);
if close == ")" && self.at_op(":") {
self.advance();
let ann = self.under(annotation_level, Self::parse_expr)?;
p.annotations.push((star_name.clone(), ann));
}
p.star = Some(star_name);
}
seen_star = true;
let _ = self.eat_op(",");
continue;
}
if self.eat_op("**") {
let name_at = self.pos;
let kw_name = self.expect_name()?;
check_dup(self, &kw_name, name_at);
seen_kwargs = true;
if close == ")" && self.at_op(":") {
self.advance();
let ann = self.under(annotation_level, Self::parse_expr)?;
p.annotations.push((kw_name.clone(), ann));
}
p.kwargs = Some(kw_name);
let _ = self.eat_op(",");
continue;
}
let name_at = self.pos;
let name = self.expect_name()?;
check_dup(self, &name, name_at);
if close == ")" && self.eat_op(":") {
let ann = self.under(annotation_level, Self::parse_expr)?;
p.annotations.push((name.clone(), ann));
}
let default = if self.eat_op("=") {
Some(self.under(default_level, Self::parse_expr)?)
} else {
None
};
if default.is_some() {
seen_default = !seen_star;
} else if seen_default && !seen_star {
return Err(self.err_span(
"parameter without a default follows parameter with a default",
name_at,
name_at,
));
}
if seen_star {
p.kwonly.push(name);
p.kwonly_defaults.push(default);
} else {
p.names.push(name);
if let Some(d) = default {
p.defaults.push(d);
}
}
if !self.eat_op(",") {
break;
}
}
Ok(p)
}
fn parse_classdef(
&mut self,
out: &mut Vec<Stmt>,
line: u32,
decorators: Vec<Expr>,
) -> Result<(), String> {
let saved = self.rule(3)?;
let rule = self.level;
let first_base = rule + 6;
self.advance(); let name = self.expect_name()?;
let type_params = self.parse_type_params()?; self.bind_type_params(out, &type_params, line);
let mut bases = Vec::new();
let mut keywords = Vec::new();
if self.eat_op("(") {
let mut order = ArgOrder::default();
while !self.at_op(")") {
let level = if bases.is_empty() && keywords.is_empty() {
first_base
} else {
first_base + 1
};
if self.eat_op("*") {
order.star().map_err(|e| self.star_error_span(e))?;
bases.push(Expr::Starred(Box::new(
self.under(level, Self::parse_expr)?,
)));
} else if self.eat_op("**") {
order.kw_unpack = true;
keywords.push(Keyword {
name: None,
value: self.under(level, Self::parse_expr)?,
});
} else if self.at_identifier()
&& matches!(&self.toks[self.pos + 1].tok, Tok::Op(o) if o == "=")
{
order.keyword = true;
let kn = self.expect_name()?;
self.expect_op("=")?;
keywords.push(Keyword {
name: Some(kn),
value: self.under(level, Self::parse_expr)?,
});
} else {
order.positional()?;
bases.push(self.under(level, Self::parse_expr)?);
}
if !self.eat_op(",") {
break;
}
}
self.expect_op(")")?;
}
let body = self.parse_scope_body("class definition", line, false)?;
self.level = saved;
out.push(Stmt::new(
StmtKind::ClassDef {
name,
bases,
keywords,
body,
decorators,
},
line,
));
Ok(())
}
fn parse_try(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let saved = self.rule(2)?;
let clause = self.level + 1;
let handler = self.level + 2;
self.advance();
let body = self.parse_suite("'try' statement", line)?;
let mut handlers = Vec::new();
while self.at_kw("except") {
let except_line = self.line();
let except_tok = self.pos;
self.advance();
let star_tok = self.pos;
let star = self.eat_op("*");
let mixed = handlers.first().is_some_and(|h: &ExceptHandler| h.star != star);
if star && (self.at_op(":") || self.at_newline()) {
return Err(if mixed {
self.err_span("invalid syntax", star_tok, star_tok)
} else {
self.err_span("expected one or more exception types", self.pos, self.pos)
});
}
let (typ, name) = if self.at_op(":") {
(None, None)
} else {
let start = self.pos;
let mut t = self.under(handler, Self::parse_expr)?;
if self.at_op(",") {
let mut types = vec![t];
while self.eat_op(",") && !self.at_op(":") && !self.at_kw("as") {
types.push(self.under(handler, Self::parse_expr)?);
}
if self.eat_kw("as") {
self.expect_name()?;
return Err(self.err_span(
"multiple exception types must be parenthesized when using 'as'",
start,
self.pos - 1,
));
}
t = Expr::Tuple(types);
}
let n = if self.eat_kw("as") {
Some(self.expect_name()?)
} else {
None
};
(Some(t), n)
};
if mixed && self.at_op(":") {
let to = if star { star_tok } else { except_tok };
return Err(self.err_span(
"cannot have both 'except' and 'except*' on the same 'try'",
except_tok,
to,
));
}
let hbody = self.under(handler, |p| {
p.parse_suite("'except' statement", except_line)
})?;
handlers.push(ExceptHandler {
typ,
name,
body: hbody,
star,
});
}
if handlers.is_empty() && !self.at_kw("finally") {
const MSG: &str = "expected 'except' or 'finally' block";
if matches!(self.cur(), Tok::Eof | Tok::Dedent) {
if let Some(nl) = self.toks[..self.pos]
.iter()
.rev()
.find(|t| matches!(t.tok, Tok::Newline))
{
return Err(at_pos(
&format!("SyntaxError: {MSG}"),
nl.line,
nl.col as i64 + 1,
nl.line,
-1,
));
}
}
return Err(self.err_here(MSG));
}
let orelse = if self.at_kw("else") {
let el = self.line();
self.advance();
self.under(clause, |p| p.parse_suite("'else' statement", el))?
} else {
Vec::new()
};
let finalbody = if self.at_kw("finally") {
let fl = self.line();
self.advance();
self.under(clause, |p| p.parse_suite("'finally' statement", fl))?
} else {
Vec::new()
};
self.level = saved;
out.push(Stmt::new(
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
},
line,
));
Ok(())
}
fn looks_like_match(&self) -> bool {
let mut i = self.pos + 1;
match self.toks.get(i).map(|t| &t.tok) {
Some(Tok::Op(o)) if o == "=" || o == ":" || o == "." || o == ";" || o == "," => {
return false
}
Some(Tok::Newline) | Some(Tok::Eof) | None => return false,
_ => {}
}
let mut depth = 0i32;
while let Some(t) = self.toks.get(i) {
match &t.tok {
Tok::Op(o) if o == "(" || o == "[" || o == "{" => depth += 1,
Tok::Op(o) if o == ")" || o == "]" || o == "}" => depth -= 1,
Tok::Op(o) if o == ":" && depth == 0 => {
let a1 = self.toks.get(i + 1).map(|t| &t.tok);
let a2 = self.toks.get(i + 2).map(|t| &t.tok);
let real_block = matches!(a1, Some(Tok::Newline))
&& matches!(a2, Some(Tok::Indent))
&& matches!(self.toks.get(i + 3).map(|t| &t.tok), Some(Tok::Name(n)) if n == "case");
let bare_header = matches!(a1, Some(Tok::Eof))
|| (matches!(a1, Some(Tok::Newline)) && !matches!(a2, Some(Tok::Indent)));
return real_block || bare_header;
}
Tok::Newline | Tok::Eof => return false,
_ => {}
}
i += 1;
}
false
}
fn parse_match(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
let saved = self.rule(2)?;
let rule = self.level;
self.advance(); let subject = self.under(rule + 1, Self::parse_exprlist)?;
self.expect_op(":")?;
self.skip_newlines();
if !matches!(self.cur(), Tok::Indent) {
return Err(format!(
"IndentationError: expected an indented block after 'match' statement on line {line}"
));
}
self.advance(); let mut cases = Vec::new();
while self.at_kw("case") {
let case_line = self.line();
self.advance(); let case_block = rule + 2;
let pattern = self.under(case_block, Self::parse_patterns)?;
let guard = if self.eat_kw("if") {
Some(self.under(case_block + 2, Self::parse_namedexpr)?)
} else {
None
};
let body = self.under(case_block, |p| p.parse_suite("'case' statement", case_line))?;
self.skip_newlines();
cases.push(MatchCase {
pattern,
guard,
body,
});
}
if matches!(self.cur(), Tok::Dedent) {
self.advance();
}
self.level = saved;
out.push(Stmt::new(StmtKind::Match { subject, cases }, line));
Ok(())
}
fn parse_patterns(&mut self) -> Result<Pattern, String> {
let start = self.pos;
let first = self.parse_pattern()?;
if self.at_op(",") {
let mut elems = vec![first];
while self.eat_op(",") {
if self.at_op(":") || self.at_kw("if") {
break;
}
elems.push(self.parse_pattern()?);
}
Ok(self.sequence_pattern(elems, start))
} else {
Ok(first)
}
}
fn parse_pattern(&mut self) -> Result<Pattern, String> {
let start = self.pos;
let p = self.parse_or_pattern()?;
if self.eat_kw("as") {
let name = self.expect_name()?;
Ok(self.pattern(PatternKind::As(Box::new(p), name), start))
} else {
Ok(p)
}
}
fn parse_or_pattern(&mut self) -> Result<Pattern, String> {
let start = self.pos;
let first = self.parse_closed_pattern()?;
if self.at_op("|") {
let mut alts = vec![first];
while self.eat_op("|") {
alts.push(self.parse_closed_pattern()?);
}
Ok(self.pattern(PatternKind::Or(alts), start))
} else {
Ok(first)
}
}
fn parse_closed_pattern(&mut self) -> Result<Pattern, String> {
let start = self.pos;
if self.eat_op("*") {
let name = if self.at_kw("_") {
self.advance();
None
} else {
Some(self.expect_name()?)
};
return Ok(self.pattern(PatternKind::Star(name), start));
}
if self.eat_op("[") {
return self.parse_sequence_pattern("]", start);
}
if self.at_op("(") {
self.advance();
if self.eat_op(")") {
return Ok(self.sequence_pattern(vec![], start));
}
let first = self.parse_pattern()?;
if self.at_op(",") {
let mut elems = vec![first];
while self.eat_op(",") {
if self.at_op(")") {
break;
}
elems.push(self.parse_pattern()?);
}
self.expect_op(")")?;
return Ok(self.sequence_pattern(elems, start));
}
self.expect_op(")")?;
return Ok(first);
}
if self.at_op("{") {
return self.parse_mapping_pattern();
}
if let Some(p) = self.try_literal_pattern()? {
return Ok(p);
}
let wildcard = self.at_kw("_");
let name = self.expect_name()?;
if wildcard {
return Ok(self.pattern(PatternKind::Wildcard, start));
}
let mut expr = Expr::Name(name);
let mut dotted = false;
while self.eat_op(".") {
let attr = self.expect_name()?;
expr = Expr::Attribute(Box::new(expr), attr);
dotted = true;
}
if self.at_op("(") {
return self.parse_class_pattern(expr, start);
}
let kind = match (&expr, dotted) {
(Expr::Name(n), false) => PatternKind::Capture(n.clone()),
_ => PatternKind::Value(expr),
};
Ok(self.pattern(kind, start))
}
fn try_literal_pattern(&mut self) -> Result<Option<Pattern>, String> {
let start = self.pos;
if self.at_op("-") {
self.advance();
let e = self.parse_atom()?;
let neg = Expr::UnaryOp(UnOp::Neg, Box::new(e));
return Ok(Some(self.pattern(PatternKind::Value(neg), start)));
}
let lit = match self.cur().clone() {
Tok::Int(_)
| Tok::BigInt(_)
| Tok::Float(_)
| Tok::Complex(_)
| Tok::Str(_)
| Tok::FString(_, _)
| Tok::TString(_, _)
| Tok::Bytes(_) => Some(self.parse_atom()?),
Tok::Name(n) if n == "None" || n == "True" || n == "False" => Some(self.parse_atom()?),
_ => None,
};
Ok(lit.map(|e| self.pattern(PatternKind::Value(e), start)))
}
fn parse_sequence_pattern(&mut self, close: &str, start: usize) -> Result<Pattern, String> {
let mut elems = Vec::new();
while !self.at_op(close) {
elems.push(self.parse_pattern()?);
if !self.eat_op(",") {
break;
}
}
self.expect_op(close)?;
Ok(self.sequence_pattern(elems, start))
}
fn sequence_pattern(&self, elems: Vec<Pattern>, start: usize) -> Pattern {
let star = elems.iter().position(|p| matches!(p.kind, PatternKind::Star(_)));
self.pattern(PatternKind::Sequence { elems, star }, start)
}
fn pattern(&self, kind: PatternKind, start: usize) -> Pattern {
Pattern {
kind,
loc: self.loc(start, self.pos - 1),
}
}
fn loc(&self, from: usize, to: usize) -> Loc {
let (a, b) = (&self.toks[from], &self.toks[to]);
Loc {
lineno: a.line,
col_offset: self.byte_col(a.line, a.col),
end_lineno: b.line,
end_col_offset: self.byte_col(b.line, b.end_col),
}
}
fn byte_col(&self, line: u32, col: u32) -> u32 {
match (line as usize).checked_sub(1).and_then(|i| self.lines.get(i)) {
Some(text) => byte_column(text, col),
None => col,
}
}
fn parse_mapping_pattern(&mut self) -> Result<Pattern, String> {
let start = self.pos;
self.advance(); let mut keys = Vec::new();
let mut rest = None;
while !self.at_op("}") {
if self.eat_op("**") {
rest = Some(self.expect_name()?);
let _ = self.eat_op(",");
break;
}
let key = self.parse_or()?;
self.expect_op(":")?;
let pat = self.parse_pattern()?;
keys.push((key, pat));
if !self.eat_op(",") {
break;
}
}
self.expect_op("}")?;
Ok(self.pattern(PatternKind::Mapping { keys, rest }, start))
}
fn parse_class_pattern(&mut self, cls: Expr, start: usize) -> Result<Pattern, String> {
self.expect_op("(")?;
let mut pos = Vec::new();
let mut kw = Vec::new();
let mut late: Vec<(usize, usize)> = Vec::new();
while !self.at_op(")") {
let start = self.pos;
if self.at_identifier()
&& matches!(&self.toks[self.pos + 1].tok, Tok::Op(o) if o == "=")
{
let kn = self.expect_name()?;
self.expect_op("=")?;
kw.push((kn, self.parse_pattern()?));
} else {
pos.push(self.parse_pattern()?);
if !kw.is_empty() {
late.push((start, self.pos - 1));
}
}
if !self.eat_op(",") {
break;
}
}
if let Some(&(first, mut last)) = late.first() {
for &(a, b) in &late[1..] {
if a != last + 2 {
break;
}
last = b;
}
return Err(self.err_span("positional patterns follow keyword patterns", first, last));
}
self.expect_op(")")?;
Ok(self.pattern(PatternKind::Class { cls, pos, kw }, start))
}
fn parse_raise(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance();
let (exc, cause) = if self.at_newline() || self.at_op(";") || matches!(self.cur(), Tok::Eof)
{
(None, None)
} else {
let raise = self.level + 1;
let e = self.under(raise, Self::parse_expr)?;
let c = if self.eat_kw("from") {
Some(self.under(raise, Self::parse_expr)?)
} else {
None
};
(Some(e), c)
};
out.push(Stmt::new(StmtKind::Raise { exc, cause }, line));
Ok(())
}
fn parse_import(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance();
self.names_after_import()?;
let mut names = Vec::new();
loop {
let mut name = self.expect_name()?;
while self.eat_op(".") {
name.push('.');
name.push_str(&self.expect_name()?);
}
let asname = if self.eat_kw("as") {
Some(self.expect_name()?)
} else {
None
};
names.push(Alias { name, asname });
if !self.eat_op(",") {
break;
}
}
out.push(Stmt::new(StmtKind::Import(names), line));
Ok(())
}
fn names_after_import(&self) -> Result<(), String> {
if !matches!(self.cur(), Tok::Newline | Tok::Eof) {
return Ok(());
}
let t = &self.toks[self.pos];
Err(at_pos(
"SyntaxError: Expected one or more names after 'import'",
t.line,
t.col as i64 + 1,
t.line,
t.col as i64 + 1,
))
}
fn parse_from_import(&mut self, out: &mut Vec<Stmt>, line: u32) -> Result<(), String> {
self.advance(); let mut level = 0;
while self.at_op(".") || self.at_op("...") {
level += if self.at_op("...") { 3 } else { 1 };
self.advance();
}
let module = if self.at_kw("import") {
None
} else {
let mut m = self.expect_name()?;
while self.eat_op(".") {
m.push('.');
m.push_str(&self.expect_name()?);
}
Some(m)
};
if !self.eat_kw("import") {
return Err(self.err_here("invalid syntax"));
}
self.names_after_import()?;
let mut names = Vec::new();
if self.eat_op("*") {
names.push(Alias {
name: "*".into(),
asname: None,
});
} else {
let paren = self.eat_op("(");
loop {
let name = self.expect_name()?;
let asname = if self.eat_kw("as") {
Some(self.expect_name()?)
} else {
None
};
names.push(Alias { name, asname });
if !self.eat_op(",") {
break;
}
if paren && self.at_op(")") {
break;
}
}
if paren {
self.expect_op(")")?;
}
}
out.push(Stmt::new(
StmtKind::ImportFrom {
module,
names,
level,
},
line,
));
Ok(())
}
fn parse_exprlist(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
let level = self.level;
let first = self.under(level + 1, Self::parse_star_expression)?;
let e = if self.at_op(",") {
let mut items = vec![first];
while self.eat_op(",") {
if self.stop_exprlist() {
break;
}
items.push(self.under(level + 3, Self::parse_star_expression)?);
}
Expr::Tuple(items)
} else {
first
};
self.level = saved;
Ok(e)
}
fn stop_exprlist(&self) -> bool {
self.at_newline()
|| matches!(self.cur(), Tok::Eof)
|| self.at_op("=")
|| self.at_op(";")
|| self.at_op(":")
|| self.at_op(")")
|| self.at_op("]")
|| self.at_op("}")
}
fn parse_star_expression(&mut self) -> Result<Expr, String> {
if self.eat_op("*") {
return Ok(Expr::Starred(Box::new(self.parse_bitor()?)));
}
self.parse_namedexpr()
}
fn parse_star_named(&mut self) -> Result<Expr, String> {
if self.eat_op("*") {
return Ok(Expr::Starred(Box::new(self.parse_bitor()?)));
}
let level = self.level + 1;
self.under(level, Self::parse_namedexpr)
}
fn parse_namedexpr(&mut self) -> Result<Expr, String> {
let start = self.pos;
let e = self.parse_ternary()?;
if self.at_op(":=") {
self.advance();
let level = self.level + 1;
let v = self.under(level, Self::parse_ternary)?;
return Ok(self.span_lines_from(Expr::NamedExpr(Box::new(e), Box::new(v)), start));
}
Ok(e)
}
fn parse_expr(&mut self) -> Result<Expr, String> {
self.parse_namedexpr()
}
fn parse_ternary(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
if self.at_kw("lambda") {
let e = self.parse_lambda()?;
self.level = saved;
return Ok(e);
}
let body_start = self.pos;
let body = self.parse_or()?;
if self.at_kw("if") {
self.advance();
let test = self.parse_or()?;
if !self.eat_kw("else") {
return Err(self.err_span(
"expected 'else' after 'if' expression",
body_start,
self.pos - 1,
));
}
let orelse = self.parse_ternary()?;
self.level = saved;
return Ok(Expr::IfExp {
test: Box::new(test),
body: Box::new(body),
orelse: Box::new(orelse),
});
}
self.level = saved;
Ok(body)
}
fn parse_lambda(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
self.advance(); let params = self.parse_params(":")?;
self.expect_op(":")?;
if self.at_kw("yield") {
return Err(self.err_here("invalid syntax"));
}
let body = self.parse_ternary()?;
self.level = saved;
Ok(Expr::Lambda {
params: Box::new(params),
body: Box::new(body),
})
}
fn parse_or(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
let level = self.level;
let mut e = self.parse_and()?;
if self.at_kw("or") {
let mut items = vec![e];
while self.eat_kw("or") {
items.push(self.under(level + 2, Self::parse_and)?);
}
e = Expr::BoolOp(BoolOp::Or, items);
}
self.level = saved;
Ok(e)
}
fn parse_and(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
let level = self.level;
let mut e = self.parse_not()?;
if self.at_kw("and") {
let mut items = vec![e];
while self.eat_kw("and") {
items.push(self.under(level + 2, Self::parse_not)?);
}
e = Expr::BoolOp(BoolOp::And, items);
}
self.level = saved;
Ok(e)
}
fn parse_not(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
let e = if self.eat_kw("not") {
Expr::UnaryOp(UnOp::Not, Box::new(self.parse_not()?))
} else {
self.parse_comparison()?
};
self.level = saved;
Ok(e)
}
fn parse_comparison(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
let right = self.level + 3;
let (sl, sc) = (self.line(), self.col());
let left = self.parse_bitor()?;
let mut ops = Vec::new();
loop {
let op = if self.at_op("<") {
CmpOp::Lt
} else if self.at_op(">") {
CmpOp::Gt
} else if self.at_op("<=") {
CmpOp::Le
} else if self.at_op(">=") {
CmpOp::Ge
} else if self.at_op("==") {
CmpOp::Eq
} else if self.at_op("!=") {
CmpOp::Ne
} else if self.at_kw("in") {
CmpOp::In
} else if self.at_kw("is") {
self.advance();
if self.eat_kw("not") {
ops.push((CmpOp::IsNot, self.under(right, Self::parse_bitor)?));
} else {
ops.push((CmpOp::Is, self.under(right, Self::parse_bitor)?));
}
continue;
} else if self.at_kw("not") {
self.advance();
if self.eat_kw("in") {
ops.push((CmpOp::NotIn, self.under(right, Self::parse_bitor)?));
continue;
} else {
return Err(format!(
"SyntaxError: expected 'in' after 'not' (line {})",
self.line()
));
}
} else {
break;
};
self.advance();
ops.push((op, self.under(right, Self::parse_bitor)?));
}
self.level = saved;
if ops.is_empty() {
Ok(left)
} else {
let end = self.prev_end_col();
Ok(spanned(
Expr::Compare(Box::new(left), ops),
sl,
sc,
end,
0,
0,
))
}
}
fn parse_bitor(&mut self) -> Result<Expr, String> {
let saved = self.rule(2)?;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_bitxor()?;
while self.at_op("|") {
e = self.binop_tail(e, BinOp::BitOr, (sl, sc), Self::parse_bitxor)?;
}
self.level = saved;
Ok(e)
}
fn parse_bitxor(&mut self) -> Result<Expr, String> {
let saved = self.rule(2)?;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_bitand()?;
while self.at_op("^") {
e = self.binop_tail(e, BinOp::BitXor, (sl, sc), Self::parse_bitand)?;
}
self.level = saved;
Ok(e)
}
fn parse_bitand(&mut self) -> Result<Expr, String> {
let saved = self.rule(2)?;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_shift()?;
while self.at_op("&") {
e = self.binop_tail(e, BinOp::BitAnd, (sl, sc), Self::parse_shift)?;
}
self.level = saved;
Ok(e)
}
fn parse_shift(&mut self) -> Result<Expr, String> {
let saved = self.rule(2)?;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_arith()?;
loop {
let op = if self.at_op("<<") {
BinOp::Shl
} else if self.at_op(">>") {
BinOp::Shr
} else {
break;
};
e = self.binop_tail(e, op, (sl, sc), Self::parse_arith)?;
}
self.level = saved;
Ok(e)
}
fn parse_arith(&mut self) -> Result<Expr, String> {
let saved = self.rule(2)?;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_term()?;
loop {
let op = if self.at_op("+") {
BinOp::Add
} else if self.at_op("-") {
BinOp::Sub
} else {
break;
};
e = self.binop_tail(e, op, (sl, sc), Self::parse_term)?;
}
self.level = saved;
Ok(e)
}
fn parse_term(&mut self) -> Result<Expr, String> {
let saved = self.rule(2)?;
let (sl, sc) = (self.line(), self.col());
let mut e = self.parse_unary()?;
loop {
let op = if self.at_op("*") {
BinOp::Mul
} else if self.at_op("/") {
BinOp::Div
} else if self.at_op("//") {
BinOp::FloorDiv
} else if self.at_op("%") {
BinOp::Mod
} else if self.at_op("@") {
BinOp::MatMul
} else {
break;
};
e = self.binop_tail(e, op, (sl, sc), Self::parse_unary)?;
}
self.level = saved;
Ok(e)
}
fn parse_unary(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
let e = self.parse_factor()?;
self.level = saved;
Ok(e)
}
fn parse_factor(&mut self) -> Result<Expr, String> {
let (sl, sc) = (self.line(), self.col());
let unary = |p: &mut Self, op: UnOp| -> Result<Expr, String> {
p.advance();
let operand = p.parse_unary()?;
let end = p.prev_end_col();
Ok(spanned(
Expr::UnaryOp(op, Box::new(operand)),
sl,
sc,
end,
0,
0,
))
};
if self.at_op("-") {
return unary(self, UnOp::Neg);
}
if self.at_op("+") {
return unary(self, UnOp::Pos);
}
if self.at_op("~") {
return unary(self, UnOp::Invert);
}
self.parse_power()
}
fn parse_power(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
let (sl, sc) = (self.line(), self.col());
let base = self.parse_await_postfix()?;
if self.at_op("**") {
let (opc, ope) = (self.col(), self.cur_end_col());
self.advance();
let exp = self.parse_unary()?; let end = self.prev_end_col();
self.level = saved;
return Ok(spanned(
Expr::BinOp(BinOp::Pow, Box::new(base), Box::new(exp)),
sl,
sc,
end,
opc,
ope,
));
}
self.level = saved;
Ok(base)
}
fn parse_await_postfix(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
let start = self.pos;
let e = if self.eat_kw("await") {
let operand = self.under(self.level - 1, Self::parse_await_postfix)?;
self.span_suspension(Expr::Await(Box::new(operand)), start)
} else {
self.parse_primary()?
};
self.level = saved;
Ok(e)
}
fn parse_primary(&mut self) -> Result<Expr, String> {
let saved = self.rule(2)?;
let raw = self.take_lead().unwrap_or(self.level);
let e = self.under(raw, |p| {
let start = (p.line(), p.col());
let atom = p.parse_atom()?;
p.parse_trailers(atom, start)
})?;
self.level = saved;
Ok(e)
}
fn take_lead(&mut self) -> Option<u32> {
let pos = self.pos;
self.lead.retain(|&(at, _)| at >= pos);
let i = self.lead.iter().position(|&(at, _)| at == pos)?;
Some(self.lead.swap_remove(i).1)
}
fn parse_trailers(
&mut self,
mut e: Expr,
(start_line, start_col): (u32, u32),
) -> Result<Expr, String> {
loop {
if self.at_op("(") {
let paren_col = self.col();
e = self.parse_call(e)?;
let end = self.prev_end_col();
e = spanned(e, start_line, start_col, end, paren_col, end);
} else if self.at_op("[") {
let bracket_col = self.col();
self.advance();
let sub = self.parse_subscript()?;
self.expect_op("]")?;
let end = self.prev_end_col();
e = spanned(
Expr::Subscript(Box::new(e), Box::new(sub)),
start_line,
start_col,
end,
bracket_col,
end,
);
} else if self.at_op(".") {
self.advance();
let attr = self.expect_name()?;
let end = self.prev_end_col();
e = spanned(
Expr::Attribute(Box::new(e), attr),
start_line,
start_col,
end,
0,
0,
);
} else {
return Ok(e);
}
}
}
fn parse_call(&mut self, func: Expr) -> Result<Expr, String> {
let raw = self.level;
self.expect_op("(")?;
let mut args = Vec::new();
let mut keywords = Vec::new();
let mut order = ArgOrder::default();
let mut kwargs: Option<u32> = None;
let kwarg_level = |kwargs: &mut Option<u32>, seen_positional: bool| match *kwargs {
Some(base) => base + 1,
None => {
let base = raw + if seen_positional { 6 } else { 5 };
*kwargs = Some(base);
base
}
};
if self.at_op(")") {
self.reach_absent_expression(raw + 3)?;
}
while !self.at_op(")") {
let first = args.is_empty() && keywords.is_empty();
if self.eat_op("*") {
order.star().map_err(|e| self.star_error_span(e))?;
let level = match kwargs {
Some(_) => kwarg_level(&mut kwargs, true),
None if first => raw + 5,
None => raw + 6,
};
args.push(Expr::Starred(Box::new(
self.under(level, Self::parse_expr)?,
)));
} else if self.eat_op("**") {
order.kw_unpack = true;
let level = kwarg_level(&mut kwargs, !first);
keywords.push(Keyword {
name: None,
value: self.under(level, Self::parse_expr)?,
});
} else if self.at_identifier()
&& matches!(&self.toks[self.pos + 1].tok, Tok::Op(o) if o == "=")
{
order.keyword = true;
let level = kwarg_level(&mut kwargs, !first);
let kn = self.expect_name()?;
self.expect_op("=")?;
let start = self.pos;
let value = self.under(level, Self::parse_expr)?;
self.comma_hint(start)?;
keywords.push(Keyword {
name: Some(kn),
value,
});
} else {
order.positional()?;
let start = self.pos;
let level = if first { raw + 2 } else { raw + 6 };
let e = self.under(level, Self::parse_namedexpr)?;
if self.at_comp_for() {
let comps = self.under(raw + 1, Self::parse_comprehension_clauses)?;
let alone = args.is_empty() && keywords.is_empty() && self.at_op(")");
if !alone {
return Err(self.err_span(
"Generator expression must be parenthesized",
start,
self.pos - 1,
));
}
args.push(Expr::GenExp(Box::new(e), comps));
} else {
self.comma_hint(start)?;
args.push(e);
}
}
if !self.eat_op(",") {
break;
}
if self.at_op(")") && kwargs.is_none() {
self.reach_absent_expression(raw + 7)?;
}
}
self.expect_op(")")?;
Ok(Expr::Call {
func: Box::new(func),
args,
keywords,
})
}
fn parse_subscript(&mut self) -> Result<Expr, String> {
let raw = self.level;
let parse_one = |p: &mut Self, slice: u32| -> Result<Expr, String> {
let lo_start = p.pos;
let lo = if p.at_op(":") {
p.reach_absent_expression(slice + 1)?;
None
} else {
Some(Box::new(p.under(slice, Self::parse_expr)?))
};
if lo.is_some() {
p.comma_hint(lo_start)?;
}
if p.at_op(":") {
p.advance();
let hi = if p.at_op(":") || p.at_op("]") || p.at_op(",") {
p.reach_absent_expression(slice + 1)?;
None
} else {
Some(Box::new(p.under(slice, Self::parse_expr)?))
};
let step = if p.eat_op(":") {
if p.at_op("]") || p.at_op(",") {
p.reach_absent_expression(slice + 2)?;
None
} else {
Some(Box::new(p.under(slice + 1, Self::parse_expr)?))
}
} else {
None
};
Ok(Expr::Slice { lo, hi, step })
} else {
Ok(*lo.unwrap())
}
};
let first = parse_one(self, raw + 2)?;
if self.at_op(",") {
let mut items = vec![first];
while self.eat_op(",") {
if self.at_op("]") {
break;
}
items.push(parse_one(self, raw + 5)?);
}
Ok(Expr::Tuple(items))
} else {
Ok(first)
}
}
fn parse_atom(&mut self) -> Result<Expr, String> {
let saved = self.rule(1)?;
let probe = match self.cur() {
Tok::Name(n) if !is_keyword(n) || matches!(n.as_str(), "True" | "False" | "None") => 0,
Tok::Ident(_) => 0,
Tok::Str(_) | Tok::FString(_, _) | Tok::TString(_, _) | Tok::Bytes(_) => 4,
_ => 1,
};
self.reach(self.level + probe)?;
let e = self.parse_atom_token()?;
self.level = saved;
Ok(e)
}
fn parse_atom_token(&mut self) -> Result<Expr, String> {
let line = self.line();
match self.cur().clone() {
Tok::Int(n) => {
self.advance();
Ok(Expr::Int(n))
}
Tok::BigInt(s) => {
self.advance();
Ok(Expr::BigInt(s))
}
Tok::Float(f) => {
self.advance();
Ok(Expr::Float(f))
}
Tok::Complex(f) => {
self.advance();
Ok(Expr::Complex(f))
}
Tok::Str(_) | Tok::FString(_, _) | Tok::TString(_, _) | Tok::Bytes(_) => {
self.parse_string_group()
}
Tok::Name(n) => {
let (nl, nc, ne) = (self.line(), self.col(), self.cur_end_col());
self.advance();
match n.as_str() {
"None" => Ok(Expr::None),
"True" => Ok(Expr::True),
"False" => Ok(Expr::False),
"lambda" => {
self.pos -= 1;
self.parse_lambda()
}
"yield" => {
let start = self.pos - 1;
let yield_expr = match self.yield_at.take() {
Some((pos, level)) if pos == start => level,
_ => self.level + 4,
};
let e = if self.eat_kw("from") {
Expr::YieldFrom(Box::new(self.under(yield_expr, Self::parse_expr)?))
} else if self.at_newline()
|| self.at_op(")")
|| self.at_op("=")
|| self.at_op(";")
|| matches!(self.cur(), Tok::Eof)
{
Expr::Yield(None)
} else {
let operand = self.under(yield_expr, Self::parse_exprlist)?;
Expr::Yield(Some(Box::new(operand)))
};
Ok(self.span_suspension(e, start))
}
_ if is_keyword(&n) => {
let _ = line;
Err(self.err_span("invalid syntax", self.pos - 1, self.pos - 1))
}
_ => Ok(spanned(Expr::Name(n), nl, nc, ne, 0, 0)),
}
}
Tok::Ident(n) => {
let (nl, nc, ne) = (self.line(), self.col(), self.cur_end_col());
self.advance();
Ok(spanned(Expr::Name(n), nl, nc, ne, 0, 0))
}
Tok::Op(o) => match o.as_str() {
"(" => self.parse_paren(),
"[" => self.parse_list(),
"{" => self.parse_brace(),
"..." => {
self.advance();
Ok(Expr::Ellipsis)
}
_ => Err(self.err_here("invalid syntax")),
},
_ => {
let _ = line;
Err(self.err_here("invalid syntax"))
}
}
}
fn parse_string_group(&mut self) -> Result<Expr, String> {
let mut parts: Vec<FStrPart> = Vec::new();
let mut any_f = false;
let mut any_t = false;
let mut any_plain = false;
let mut byte_acc: Option<Vec<u8>> = None;
loop {
match self.cur().clone() {
Tok::Str(s) => {
self.advance();
any_plain = true;
parts.push(FStrPart::Lit(s));
}
Tok::Bytes(b) => {
self.advance();
byte_acc.get_or_insert_with(Vec::new).extend(b);
}
Tok::FString(raw, is_raw) => {
self.advance();
any_f = true;
let mut sub = self.parse_fstring(&raw, is_raw)?;
parts.append(&mut sub);
}
Tok::TString(raw, is_raw) => {
self.advance();
any_t = true;
let mut sub = self.parse_fstring(&raw, is_raw)?;
parts.append(&mut sub);
}
_ => break,
}
}
if any_t && (any_f || any_plain || byte_acc.is_some()) {
return Err(
"SyntaxError: cannot mix t-string literals with string or bytes literals"
.to_string(),
);
}
if byte_acc.is_some() && (any_plain || any_f || any_t) {
return Err("SyntaxError: cannot mix bytes and nonbytes literals".to_string());
}
if let Some(b) = byte_acc {
return Ok(Expr::Bytes(b));
}
if any_t {
Ok(Expr::TString(parts))
} else if any_f {
Ok(Expr::FString(parts))
} else {
let mut s = String::new();
for p in parts {
if let FStrPart::Lit(l) = p {
s.push_str(&l);
}
}
Ok(Expr::Str(s))
}
}
fn parse_fstring(&self, raw: &str, is_raw: bool) -> Result<Vec<FStrPart>, String> {
let chars: Vec<char> = raw.chars().collect();
let mut parts = Vec::new();
let mut lit = String::new();
let mut i = 0;
while i < chars.len() {
let c = chars[i];
if c == '{' {
if chars.get(i + 1) == Some(&'{') {
lit.push('{');
i += 2;
continue;
}
if crate::lexer::ends_with_named_escape_lead(&lit, is_raw) {
lit.push('{');
i += 1;
while i < chars.len() && chars[i] != '}' {
lit.push(chars[i]);
i += 1;
}
if i < chars.len() {
lit.push('}');
i += 1;
}
continue;
}
if !lit.is_empty() {
let decoded = crate::lexer::decode_escapes(&lit, is_raw)?;
parts.push(FStrPart::Lit(decoded));
lit.clear();
}
let mut depth = 1;
i += 1;
let mut field = String::new();
while i < chars.len() && depth > 0 {
match chars[i] {
'{' => {
depth += 1;
field.push('{');
}
'}' => {
depth -= 1;
if depth > 0 {
field.push('}');
}
}
other => field.push(other),
}
i += 1;
}
parts.extend(self.build_fstring_field(&field, is_raw)?);
} else if c == '}' {
if chars.get(i + 1) == Some(&'}') {
lit.push('}');
i += 2;
continue;
}
lit.push('}');
i += 1;
} else {
lit.push(c);
i += 1;
}
}
if !lit.is_empty() {
let decoded = crate::lexer::decode_escapes(&lit, is_raw)?;
parts.push(FStrPart::Lit(decoded));
}
Ok(parts)
}
fn build_fstring_field(&self, field: &str, is_raw: bool) -> Result<Vec<FStrPart>, String> {
let mut debug_prefix: Option<String> = None;
let work: String = if let Some(eq) = find_debug_eq(field) {
let after = &field[eq + 1..];
let ws = after.len() - after.trim_start().len();
debug_prefix = Some(field[..eq + 1 + ws].to_string());
format!("{}{}", &field[..eq], after.trim_start())
} else {
field.to_string()
};
let field: &str = &work;
let mut expr_src = field;
let mut spec: Vec<FStrPart> = Vec::new();
let mut conv: Option<char> = None;
if let Some(idx) = find_top_level(field, ':') {
spec = self.parse_fstring(&field[idx + 1..], is_raw)?;
expr_src = &field[..idx];
}
if expr_src.len() >= 2 {
let bytes = expr_src.as_bytes();
if bytes[expr_src.len() - 2] == b'!' {
let c = bytes[expr_src.len() - 1] as char;
if matches!(c, 's' | 'r' | 'a') {
conv = Some(c);
expr_src = &expr_src[..expr_src.len() - 2];
}
}
}
if debug_prefix.is_some() && conv.is_none() && spec.is_empty() {
conv = Some('r');
}
let expr_src = expr_src.trim();
let sub =
parse_source(&format!("({expr_src})"), self.level.saturating_sub(7)).map_err(|e| {
if e == TOO_COMPLEX {
e
} else {
format!("f-string: {e}")
}
})?;
let expr = match sub.into_iter().next() {
Some(Stmt {
kind: StmtKind::Expr(e),
..
}) => e,
_ => return Err(format!("f-string: invalid expression {{{expr_src}}}")),
};
let mut out = Vec::with_capacity(2);
if let Some(pre) = debug_prefix {
out.push(FStrPart::Lit(pre));
}
out.push(FStrPart::Expr {
expr: Box::new(expr),
src: expr_src.to_string(),
conv,
spec,
});
Ok(out)
}
fn parse_paren(&mut self) -> Result<Expr, String> {
let atom = self.level;
let open = self.pos;
self.advance(); if self.eat_op(")") {
self.reach_absent_expression(atom + 6)?;
return Ok(Expr::Tuple(Vec::new()));
}
let start = self.pos;
if self.at_kw("yield") {
self.yield_at = Some((self.pos, atom + 4));
}
let first = self.under(atom + 4, Self::parse_star_named)?;
if self.at_comp_for() {
let comps = self.under(atom + 2, Self::parse_comprehension_clauses)?;
self.expect_op(")")?;
return Ok(Expr::GenExp(Box::new(first), comps));
}
self.comma_hint(start)?;
if self.at_op(",") {
let mut items = vec![first];
while self.eat_op(",") {
let level = if items.len() == 1 { atom + 6 } else { atom + 7 };
if self.at_op(")") {
self.reach_absent_expression(level + 2)?;
break;
}
let start = self.pos;
items.push(self.under(level, Self::parse_star_named)?);
self.comma_hint(start)?;
}
self.expect_op(")")?;
return Ok(Expr::Tuple(items));
}
self.expect_op(")")?;
let (line, col) = self.node_start(start, self.pos - 1);
self.groups.insert(open, (self.pos - 1, line, col));
Ok(first)
}
fn parse_list(&mut self) -> Result<Expr, String> {
let atom = self.level;
let open = self.pos;
self.advance(); if self.eat_op("]") {
self.reach_absent_expression(atom + 7)?;
return Ok(Expr::List(Vec::new()));
}
let start = self.pos;
let first = self.under(atom + 5, Self::parse_star_named)?;
if self.at_comp_for() {
let comps = self.under(atom + 2, Self::parse_comprehension_clauses)?;
self.expect_op("]")?;
return Ok(self.span_lines_from(Expr::ListComp(Box::new(first), comps), open));
}
self.comma_hint(start)?;
let mut items = vec![first];
while self.eat_op(",") {
if self.at_op("]") {
self.reach_absent_expression(atom + 8)?;
break;
}
let start = self.pos;
items.push(self.under(atom + 6, Self::parse_star_named)?);
self.comma_hint(start)?;
}
self.expect_op("]")?;
Ok(Expr::List(items))
}
fn parse_brace(&mut self) -> Result<Expr, String> {
let (open, line, start) = (self.pos, self.line(), self.col());
let e = self.parse_brace_display()?;
Ok(match e {
Expr::Dict(_) | Expr::Set(_) => spanned(e, line, start, self.prev_end_col(), 0, 0),
Expr::DictComp(..) | Expr::SetComp(..) => self.span_lines_from(e, open),
other => other,
})
}
fn parse_brace_display(&mut self) -> Result<Expr, String> {
let atom = self.level;
self.advance(); if self.eat_op("}") {
self.reach_absent_expression(atom + 7)?;
return Ok(Expr::Dict(Vec::new()));
}
if self.eat_op("**") {
let v = self.under(atom + 5, Self::parse_expr)?;
let mut pairs = vec![(None, v)];
while self.eat_op(",") {
if self.at_op("}") {
break;
}
if self.eat_op("**") {
pairs.push((None, self.under(atom + 6, Self::parse_expr)?));
} else {
let k = self.under(atom + 7, Self::parse_expr)?;
self.expect_op(":")?;
pairs.push((Some(k), self.under(atom + 7, Self::parse_expr)?));
}
}
self.expect_op("}")?;
return Ok(Expr::Dict(pairs));
}
let first_start = self.pos;
let first = self.under(atom + 5, Self::parse_star_named)?;
if self.at_op(":") {
self.advance();
let v = self.under(atom + 6, Self::parse_expr)?;
if self.at_comp_for() {
let comps = self.under(atom + 2, Self::parse_comprehension_clauses)?;
self.expect_op("}")?;
return Ok(Expr::DictComp(Box::new(first), Box::new(v), comps));
}
let mut pairs = vec![(Some(first), v)];
while self.eat_op(",") {
if self.at_op("}") {
self.reach_absent_expression(atom + 8)?;
break;
}
if self.eat_op("**") {
pairs.push((None, self.under(atom + 6, Self::parse_expr)?));
continue;
}
let k_start = self.pos;
let k = self.under(atom + 7, Self::parse_expr)?;
if !self.at_op(":") {
let t = &self.toks[k_start];
return Err(at_pos(
"SyntaxError: ':' expected after dictionary key",
t.line,
t.col as i64 + 1,
t.line,
0,
));
}
self.advance();
pairs.push((Some(k), self.under(atom + 7, Self::parse_expr)?));
}
self.expect_op("}")?;
Ok(Expr::Dict(pairs))
} else if self.at_comp_for() {
let comps = self.under(atom + 2, Self::parse_comprehension_clauses)?;
self.expect_op("}")?;
Ok(Expr::SetComp(Box::new(first), comps))
} else {
self.comma_hint(first_start)?;
let mut items = vec![first];
while self.eat_op(",") {
if self.at_op("}") {
self.reach_absent_expression(atom + 8)?;
break;
}
let start = self.pos;
items.push(self.under(atom + 6, Self::parse_star_named)?);
self.comma_hint(start)?;
}
self.expect_op("}")?;
Ok(Expr::Set(items))
}
}
fn at_comp_for(&self) -> bool {
self.at_kw("for") || self.at_kw("async")
}
fn parse_comprehension_clauses(&mut self) -> Result<Vec<Comprehension>, String> {
let clause = self.level + 3;
let mut comps = Vec::new();
while self.at_kw("for") || self.at_kw("async") {
let is_async = self.eat_kw("async");
self.advance(); let start = self.pos;
let target = self.parse_target_tuple()?;
self.check_for_target(&target, start, true)?;
self.advance(); let iter = self.under(clause, Self::parse_or)?;
let mut ifs = Vec::new();
while self.at_kw("if") {
self.advance();
ifs.push(self.under(clause + 2, Self::parse_or)?);
}
comps.push(Comprehension {
target: Box::new(target),
iter: Box::new(iter),
ifs,
is_async,
});
}
Ok(comps)
}
}
fn find_top_level(s: &str, ch: char) -> Option<usize> {
let mut depth = 0i32;
for (i, c) in s.char_indices() {
match c {
'(' | '[' | '{' => depth += 1,
')' | ']' | '}' => depth -= 1,
_ if c == ch && depth == 0 => return Some(i),
_ => {}
}
}
None
}
fn find_debug_eq(field: &str) -> Option<usize> {
let bytes = field.as_bytes();
let mut depth = 0i32;
let mut quote: Option<u8> = None;
let mut i = 0;
while i < bytes.len() {
let c = bytes[i];
if let Some(q) = quote {
if c == b'\\' {
i += 2;
continue;
}
if c == q {
quote = None;
}
i += 1;
continue;
}
match c {
b'\'' | b'"' => quote = Some(c),
b'(' | b'[' | b'{' => depth += 1,
b')' | b']' | b'}' => depth -= 1,
b':' if depth == 0 => return None,
b'=' if depth == 0 => {
let next = bytes.get(i + 1).copied();
let prev = if i > 0 {
bytes.get(i - 1).copied()
} else {
None
};
let is_eqeq = next == Some(b'=');
let is_cmp = matches!(
prev,
Some(b'=') | Some(b'!') | Some(b'<') | Some(b'>') | Some(b':')
);
if !is_eqeq && !is_cmp {
return Some(i);
}
}
_ => {}
}
i += 1;
}
None
}
fn augassign_op(o: &str) -> Option<BinOp> {
Some(match o {
"+=" => BinOp::Add,
"-=" => BinOp::Sub,
"*=" => BinOp::Mul,
"/=" => BinOp::Div,
"//=" => BinOp::FloorDiv,
"%=" => BinOp::Mod,
"**=" => BinOp::Pow,
"&=" => BinOp::BitAnd,
"|=" => BinOp::BitOr,
"^=" => BinOp::BitXor,
"<<=" => BinOp::Shl,
">>=" => BinOp::Shr,
"@=" => BinOp::MatMul,
_ => return None,
})
}
fn unspan(e: &Expr) -> &Expr {
match e {
Expr::Spanned(inner, _) => unspan(inner),
other => other,
}
}
fn expr_name(e: &Expr) -> &'static str {
match unspan(e) {
Expr::None => "None",
Expr::True => "True",
Expr::False => "False",
Expr::Ellipsis => "ellipsis",
Expr::Int(_)
| Expr::BigInt(_)
| Expr::Float(_)
| Expr::Complex(_)
| Expr::Str(_)
| Expr::Bytes(_) => "literal",
Expr::FString(_) => "f-string expression",
Expr::TString(_) => "t-string expression",
Expr::Name(_) => "name",
Expr::List(_) => "list",
Expr::Tuple(_) => "tuple",
Expr::Set(_) => "set display",
Expr::Dict(_) => "dict literal",
Expr::Starred(_) => "starred",
Expr::Compare(..) => "comparison",
Expr::IfExp { .. } => "conditional expression",
Expr::Call { .. } => "function call",
Expr::Attribute(..) => "attribute",
Expr::Subscript(..) => "subscript",
Expr::Slice { .. } => "slice",
Expr::Lambda { .. } => "lambda",
Expr::ListComp(..) => "list comprehension",
Expr::SetComp(..) => "set comprehension",
Expr::DictComp(..) => "dict comprehension",
Expr::GenExp(..) => "generator expression",
Expr::Yield(_) | Expr::YieldFrom(_) => "yield expression",
Expr::Await(_) => "await expression",
Expr::NamedExpr(..) => "named expression",
_ => "expression",
}
}