use crate::ast::*;
use crate::host::{binop as bop, iop, ops, unop, FuncDef, HandlerDef, TryDef};
use fusevm::{Chunk, ChunkBuilder, Op, Value};
use std::collections::{HashMap, HashSet};
#[derive(Default)]
pub struct Program {
pub main: Chunk,
pub functions: Vec<(String, FuncDef)>,
pub procs: Vec<FuncDef>,
pub tries: Vec<TryDef>,
pub warnings: Vec<(u32, String)>,
pub positions: Vec<(u64, Vec<Span>)>,
}
pub fn rebase_program(prog: &mut Program, func_off: usize, try_off: usize) {
if func_off == 0 && try_off == 0 {
return;
}
rebase_chunk(&mut prog.main, func_off, try_off);
for (_, f) in &mut prog.functions {
rebase_chunk(&mut f.chunk, func_off, try_off);
}
for t in &mut prog.tries {
rebase_chunk(&mut t.body, func_off, try_off);
for h in &mut t.handlers {
let (tc, hb) = (&mut h.typ, &mut h.body);
if let Some(tc) = tc {
rebase_chunk(tc, func_off, try_off);
}
rebase_chunk(hb, func_off, try_off);
}
if let Some(e) = &mut t.orelse {
rebase_chunk(e, func_off, try_off);
}
if let Some(f) = &mut t.finalbody {
rebase_chunk(f, func_off, try_off);
}
}
}
fn rebase_chunk(chunk: &mut Chunk, func_off: usize, try_off: usize) {
for i in 1..chunk.ops.len() {
let off = match chunk.ops[i] {
Op::CallBuiltin(id, _) if id == ops::MKFUNC || id == ops::MKLAMBDA => func_off,
Op::CallBuiltin(id, 1) if id == ops::TRY || id == ops::LOOP_BODY => try_off,
_ => continue,
};
if off == 0 {
continue;
}
if let Op::LoadInt(v) = &mut chunk.ops[i - 1] {
*v += off as i64;
}
}
for sub in &mut chunk.sub_chunks {
rebase_chunk(sub, func_off, try_off);
}
}
struct LoopCtx {
breaks: Vec<usize>,
continues: Vec<usize>,
signal: bool,
}
#[derive(Default)]
pub struct Compiler {
functions: Vec<(String, FuncDef)>,
tries: Vec<TryDef>,
loops: Vec<LoopCtx>,
native_slots: Option<HashMap<String, u16>>,
native_next_slot: u16,
fn_slots: HashMap<String, u16>,
fn_slots_bound: HashSet<String>,
tmp: usize,
debug: bool,
cur_line: u32,
fn_depth: usize,
func_scopes: Vec<HashSet<String>>,
qual_prefix: String,
warnings: Vec<(u32, String)>,
interactive: bool,
in_class_body: bool,
def_depth: usize,
await_scope: AwaitScope,
in_comprehension: bool,
node_span: Span,
suppress_hint: bool,
positions: Vec<Vec<Span>>,
collected_positions: Vec<(u64, Vec<Span>)>,
}
#[derive(Clone, Copy, PartialEq)]
enum ScopeKind {
Function,
Comprehension,
ClassBody,
}
impl ScopeKind {
fn is_function(self) -> bool {
self != ScopeKind::ClassBody
}
}
#[derive(Clone, Copy, PartialEq, Default)]
enum AwaitScope {
#[default]
Module,
Sync,
Async,
}
pub fn compile(stmts: &[Stmt], debug: bool) -> Result<Program, String> {
compile_ex(stmts, debug, false)
}
pub fn compile_interactive(stmts: &[Stmt]) -> Result<Program, String> {
compile_ex(stmts, false, true)
}
fn compile_ex(stmts: &[Stmt], debug: bool, interactive: bool) -> Result<Program, String> {
let mut owned;
let stmts = if crate::mangle::has_class(stmts) {
owned = stmts.to_vec();
crate::mangle::mangle_body("_", &mut owned);
&owned[..]
} else {
stmts
};
let mut c = Compiler {
debug,
interactive,
..Default::default()
};
let mut b = ChunkBuilder::new();
c.begin_chunk();
if let Some(doc) = docstring(stmts) {
let store = Stmt::from(StmtKind::Assign {
targets: vec![Expr::Name("__doc__".to_string())],
value: Expr::Str(doc),
});
c.compile_stmts(&mut b, std::slice::from_ref(&store))?;
}
c.compile_stmts(&mut b, stmts)?;
let main = c.finish_chunk(b);
Ok(Program {
main,
functions: c.functions,
procs: Vec::new(),
tries: c.tries,
warnings: c.warnings,
positions: c.collected_positions,
})
}
fn argc(n: usize) -> Result<u8, String> {
u8::try_from(n).map_err(|_| "too many arguments (>255) for one call".to_string())
}
#[derive(Clone, Copy)]
struct NativeLoopRange {
target_slot: u16,
stop_slot: u16,
step: i64,
}
#[derive(Clone, Copy)]
struct NativeLoopNs<'a> {
loads: &'a [(String, u16)],
writebacks: &'a [(String, u16)],
guard_slots: &'a [u16],
}
impl NativeLoopNs<'static> {
const EMPTY: Self = Self {
loads: &[],
writebacks: &[],
guard_slots: &[],
};
}
impl Compiler {
fn begin_chunk(&mut self) {
self.positions.push(Vec::new());
}
fn finish_chunk(&mut self, b: ChunkBuilder) -> Chunk {
let c = b.build();
let mut frame = self.positions.pop().unwrap_or_default();
frame.resize(c.ops.len(), Span::NONE);
self.collected_positions.push((c.op_hash, frame));
c
}
fn record_span(&mut self, idx: usize) {
let sp = self.node_span;
if !sp.is_some() {
return;
}
if let Some(frame) = self.positions.last_mut() {
if frame.len() <= idx {
frame.resize(idx + 1, Span::NONE);
}
frame[idx] = sp;
}
}
fn name_const(&self, b: &mut ChunkBuilder, s: &str) {
let k = b.add_constant(Value::str(s));
b.emit(Op::LoadConst(k), 0);
}
fn strlit(&self, b: &mut ChunkBuilder, s: &str) {
let k = b.add_constant(Value::str(s));
b.emit(Op::LoadConst(k), 0);
b.emit(Op::CallBuiltin(ops::MKSTR, 1), 0);
}
fn compile_stmts(&mut self, b: &mut ChunkBuilder, stmts: &[Stmt]) -> Result<(), String> {
for s in stmts {
self.compile_stmt(b, s)?;
}
Ok(())
}
fn compile_stmt(&mut self, b: &mut ChunkBuilder, s: &Stmt) -> Result<(), String> {
if s.line != 0 {
self.cur_line = s.line;
}
if self.debug && s.line != 0 {
b.emit(Op::LoadInt(s.line as i64), s.line);
b.emit(Op::CallBuiltin(ops::DBG_LINE, 1), s.line);
b.emit(Op::Pop, s.line);
}
let line = s.line;
match &s.kind {
StmtKind::Expr(e) => {
self.compile_expr(b, e)?;
if self.interactive && self.fn_depth == 0 {
b.emit(Op::CallBuiltin(ops::DISPLAYHOOK, 1), line);
b.emit(Op::Pop, line);
} else {
b.emit(Op::Pop, line);
}
}
StmtKind::Pass => {}
StmtKind::Assign { targets, value } => {
self.suppress_hint = targets.len() == 1
&& matches!(targets[0].unspanned(), Expr::Name(_))
&& matches!(value.unspanned(), Expr::Call { .. });
self.compile_expr(b, value)?;
self.suppress_hint = false;
for (i, t) in targets.iter().enumerate() {
if i + 1 < targets.len() {
b.emit(Op::Dup, line);
}
self.compile_assign(b, t)?;
}
}
StmtKind::AnnAssign {
target,
annotation,
value,
} => {
if self.in_class_body {
if let Expr::Name(n) = target.unspanned() {
self.name_const(b, "__annotations__");
b.emit(Op::CallBuiltin(ops::GETLOCAL, 1), self.cur_line); self.strlit(b, n); let ann_body = vec![Stmt::from(StmtKind::Return(Some(annotation.clone())))];
let empty = Params::default();
self.fn_depth += 1;
let thunk_id = self.build_function("<annotate>", &empty, &ann_body);
self.fn_depth -= 1;
self.emit_make_func(b, thunk_id?, &empty)?; b.emit(Op::CallBuiltin(ops::TRY_ANNOTATION, 3), 0);
}
}
if let Some(v) = value {
self.compile_expr(b, v)?;
self.compile_assign(b, target)?;
}
}
StmtKind::AugAssign { target, op, value } => {
self.compile_augassign(b, target, *op, value)?;
}
StmtKind::If { test, body, orelse } => {
self.compile_if(b, test, body, orelse)?;
}
StmtKind::While { test, body, orelse } => {
self.compile_while(b, test, body, orelse)?;
}
StmtKind::For {
target,
iter,
body,
orelse,
is_async,
} => {
if *is_async {
self.compile_async_for(b, target, iter, body, orelse)?;
} else {
self.compile_for(b, target, iter, body, orelse)?;
}
}
StmtKind::FuncDef {
name,
params,
body,
decorators,
is_async,
} => {
self.compile_funcdef(b, name, params, body, decorators, *is_async)?;
}
StmtKind::ClassDef {
name,
bases,
keywords,
body,
decorators,
} => {
self.compile_classdef(b, name, bases, keywords, body, decorators)?;
}
StmtKind::Return(e) => {
if self.def_depth == 0 {
return Err("SyntaxError: 'return' outside function".to_string());
}
match e {
Some(e) => {
self.suppress_hint = matches!(
e.unspanned(),
Expr::Call { func, .. } if matches!(func.unspanned(), Expr::Name(_))
);
self.compile_expr(b, e)?;
self.suppress_hint = false;
}
None => {
b.emit(Op::LoadUndef, line);
}
}
b.emit(Op::CallBuiltin(ops::SIG_RETURN, 1), line);
}
StmtKind::Break => {
let lc = self
.loops
.last_mut()
.ok_or("SyntaxError: 'break' outside loop")?;
if lc.signal {
b.emit(Op::CallBuiltin(ops::SIG_BREAK, 0), line);
b.emit(Op::Pop, line);
} else {
let j = b.emit(Op::Jump(0), line);
self.loops.last_mut().unwrap().breaks.push(j);
}
}
StmtKind::Continue => {
let lc = self
.loops
.last_mut()
.ok_or("SyntaxError: 'continue' not properly in loop")?;
if lc.signal {
b.emit(Op::CallBuiltin(ops::SIG_CONTINUE, 0), line);
b.emit(Op::Pop, line);
} else {
let j = b.emit(Op::Jump(0), line);
self.loops.last_mut().unwrap().continues.push(j);
}
}
StmtKind::Delete(targets) => {
for t in targets {
self.compile_delete(b, t)?;
}
}
StmtKind::Global(names) => {
for n in names {
self.name_const(b, n);
b.emit(Op::CallBuiltin(ops::DECLARE_GLOBAL, 1), line);
b.emit(Op::Pop, line);
}
}
StmtKind::Nonlocal(names) => {
for n in names {
self.check_nonlocal_binding(n)?;
self.name_const(b, n);
b.emit(Op::CallBuiltin(ops::DECLARE_NONLOCAL, 1), line);
b.emit(Op::Pop, line);
}
}
StmtKind::Match { subject, cases } => {
self.compile_match(b, subject, cases)?;
}
StmtKind::Raise { exc, cause } => match exc {
Some(e) => match cause {
Some(c) => {
self.compile_expr(b, c)?;
self.compile_expr(b, e)?;
b.emit(Op::CallBuiltin(ops::RAISE, 2), line);
}
None => {
self.compile_expr(b, e)?;
b.emit(Op::CallBuiltin(ops::RAISE, 1), line);
}
},
None => {
b.emit(Op::CallBuiltin(ops::RERAISE, 0), line);
b.emit(Op::Pop, line);
}
},
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
} => {
self.compile_try(b, body, handlers, orelse, finalbody)?;
}
StmtKind::With {
items,
body,
is_async,
} => {
self.compile_with(b, items, body, *is_async)?;
}
StmtKind::Assert { test, msg } => {
self.compile_assert(b, test, msg)?;
}
StmtKind::Import(aliases) => {
for a in aliases {
match &a.asname {
Some(asname) => {
self.strlit(b, &a.name);
b.emit(Op::CallBuiltin(ops::IMPORT, 1), line);
self.store_name(b, asname);
}
None => {
let top = a.name.split('.').next().unwrap_or(&a.name).to_string();
self.strlit(b, &a.name);
b.emit(Op::CallBuiltin(ops::IMPORT, 1), line);
if top != a.name {
b.emit(Op::Pop, 0);
self.strlit(b, &top);
b.emit(Op::CallBuiltin(ops::IMPORT, 1), line);
}
self.store_name(b, &top);
}
}
}
}
StmtKind::ImportFrom {
module,
names,
level,
} => {
let m = module.clone().unwrap_or_default();
for a in names {
if *level > 0 {
if a.name == "*" {
b.emit(Op::LoadInt(*level as i64), line);
self.strlit(b, &m);
self.strlit(b, "*");
b.emit(Op::CallBuiltin(ops::IMPORT_RELATIVE, 3), line);
b.emit(Op::CallBuiltin(ops::IMPORT_STAR, 1), line);
b.emit(Op::Pop, 0);
continue;
}
b.emit(Op::LoadInt(*level as i64), line);
self.strlit(b, &m);
self.strlit(b, &a.name);
b.emit(Op::CallBuiltin(ops::IMPORT_RELATIVE, 3), line);
let bind = a.asname.clone().unwrap_or_else(|| a.name.clone());
self.store_name(b, &bind);
continue;
}
if a.name == "*" {
self.strlit(b, &m);
b.emit(Op::CallBuiltin(ops::IMPORT, 1), line);
b.emit(Op::CallBuiltin(ops::IMPORT_STAR, 1), line);
b.emit(Op::Pop, 0);
continue;
}
self.strlit(b, &m);
b.emit(Op::CallBuiltin(ops::IMPORT, 1), line);
self.strlit(b, &a.name);
b.emit(Op::CallBuiltin(ops::IMPORT_FROM, 2), line);
let bind = a.asname.clone().unwrap_or_else(|| a.name.clone());
self.store_name(b, &bind);
}
}
}
Ok(())
}
fn slot_of(&self, name: &str) -> Option<u16> {
if let Some(s) = self.native_slots.as_ref().and_then(|m| m.get(name)) {
return Some(*s);
}
self.fn_slots.get(name).copied()
}
fn store_name(&self, b: &mut ChunkBuilder, name: &str) {
if let Some(slot) = self.slot_of(name) {
b.emit(Op::SetSlot(slot), 0); return;
}
self.name_const(b, name);
b.emit(Op::Swap, 0);
b.emit(Op::CallBuiltin(ops::SETLOCAL, 2), 0);
b.emit(Op::Pop, 0);
}
fn compile_assign(&mut self, b: &mut ChunkBuilder, target: &Expr) -> Result<(), String> {
if let Expr::Spanned(inner, sp) = target {
let prev = std::mem::replace(&mut self.node_span, *sp);
let r = self.compile_assign(b, inner);
self.node_span = prev;
return r;
}
let target = target.unspanned();
match target {
Expr::Name(n) => {
if let Some(slot) = self.slot_of(n) {
b.emit(Op::SetSlot(slot), 0); } else {
self.name_const(b, n);
b.emit(Op::Swap, 0);
b.emit(Op::CallBuiltin(ops::SETLOCAL, 2), 0);
b.emit(Op::Pop, 0);
}
}
Expr::Attribute(recv, attr) => {
self.compile_expr(b, recv)?; self.name_const(b, attr); b.emit(Op::Rot, 0); let op = b.emit(Op::CallBuiltin(ops::SETATTR, 3), self.cur_line);
self.record_span(op);
b.emit(Op::Pop, 0);
}
Expr::Subscript(recv, idx) => {
self.compile_expr(b, recv)?; self.compile_subscript_index(b, idx)?; b.emit(Op::Rot, 0); let op = b.emit(Op::CallBuiltin(ops::SETITEM, 3), self.cur_line);
self.record_span(op);
b.emit(Op::Pop, 0);
}
Expr::Tuple(items) | Expr::List(items) => {
self.compile_unpack_assign(b, items)?;
}
Expr::Starred(inner) => self.compile_assign(b, inner)?,
_ => return Err("SyntaxError: cannot assign to this expression".into()),
}
Ok(())
}
fn compile_unpack_assign(
&mut self,
b: &mut ChunkBuilder,
items: &[Expr],
) -> Result<(), String> {
let star_idx = items
.iter()
.position(|e| matches!(e, Expr::Starred(_)))
.map(|i| i as i64)
.unwrap_or(-1);
b.emit(Op::LoadInt(items.len() as i64), 0);
b.emit(Op::LoadInt(star_idx), 0);
b.emit(Op::CallBuiltin(ops::UNPACK, 3), 0);
for t in items {
self.compile_assign(b, t)?;
}
Ok(())
}
fn compile_delete(&mut self, b: &mut ChunkBuilder, target: &Expr) -> Result<(), String> {
if let Expr::Spanned(inner, sp) = target {
let prev = std::mem::replace(&mut self.node_span, *sp);
let r = self.compile_delete(b, inner);
self.node_span = prev;
return r;
}
let target = target.unspanned();
match target {
Expr::Name(n) => {
self.name_const(b, n);
b.emit(Op::CallBuiltin(ops::DELNAME, 1), 0);
b.emit(Op::Pop, 0);
}
Expr::Subscript(recv, idx) => {
self.compile_expr(b, recv)?;
self.compile_subscript_index(b, idx)?;
let op = b.emit(Op::CallBuiltin(ops::DELITEM, 2), self.cur_line);
self.record_span(op);
b.emit(Op::Pop, 0);
}
Expr::Attribute(recv, attr) => {
self.compile_expr(b, recv)?;
self.name_const(b, attr);
let op = b.emit(Op::CallBuiltin(ops::DELATTR, 2), self.cur_line);
self.record_span(op);
b.emit(Op::Pop, 0);
}
_ => return Err("SyntaxError: cannot delete this expression".into()),
}
Ok(())
}
fn iop_tag(op: BinOp) -> i64 {
use iop::*;
match op {
BinOp::Add => ADD,
BinOp::Sub => SUB,
BinOp::Mul => MUL,
BinOp::Div => DIV,
BinOp::FloorDiv => FLOORDIV,
BinOp::Mod => MOD,
BinOp::Pow => POW,
BinOp::MatMul => MATMUL,
BinOp::BitAnd => BITAND,
BinOp::BitOr => BITOR,
BinOp::BitXor => BITXOR,
BinOp::Shl => SHL,
BinOp::Shr => SHR,
}
}
fn compile_augassign(
&mut self,
b: &mut ChunkBuilder,
target: &Expr,
op: BinOp,
value: &Expr,
) -> Result<(), String> {
let tag = Self::iop_tag(op);
match target.unspanned() {
Expr::Name(n) => {
if let (Some(slot), Some(nop)) = (
self.native_slots.as_ref().and_then(|m| m.get(n).copied()),
native_arith_op(op),
) {
self.compile_expr(b, target)?; self.compile_expr(b, value)?;
b.emit(nop, self.cur_line);
b.emit(Op::SetSlot(slot), 0);
return Ok(());
}
b.emit(Op::LoadInt(tag), 0);
self.compile_expr(b, target)?;
self.compile_expr(b, value)?;
b.emit(Op::CallBuiltin(ops::INPLACE, 3), self.cur_line);
self.compile_assign(b, target)?;
}
Expr::Attribute(recv, attr) => {
let rt = format!(".aug{}", self.tmp);
self.tmp += 1;
self.compile_expr(b, recv)?;
self.compile_assign(b, &Expr::Name(rt.clone()))?;
let lval = Expr::Attribute(Box::new(Expr::Name(rt.clone())), attr.clone());
b.emit(Op::LoadInt(tag), 0);
self.compile_expr(b, &lval)?;
self.compile_expr(b, value)?;
b.emit(Op::CallBuiltin(ops::INPLACE, 3), self.cur_line);
self.compile_assign(b, &lval)?;
}
Expr::Subscript(recv, idx) => {
let rt = format!(".aug{}", self.tmp);
self.tmp += 1;
let it = format!(".aug{}", self.tmp);
self.tmp += 1;
self.compile_expr(b, recv)?;
self.compile_assign(b, &Expr::Name(rt.clone()))?;
self.compile_subscript_index(b, idx)?;
self.compile_assign(b, &Expr::Name(it.clone()))?;
let lval = Expr::Subscript(
Box::new(Expr::Name(rt.clone())),
Box::new(Expr::Name(it.clone())),
);
b.emit(Op::LoadInt(tag), 0);
self.compile_expr(b, &lval)?;
self.compile_expr(b, value)?;
b.emit(Op::CallBuiltin(ops::INPLACE, 3), self.cur_line);
self.compile_assign(b, &lval)?;
}
_ => return Err("SyntaxError: cannot assign to this expression".into()),
}
Ok(())
}
fn compile_condition(&mut self, b: &mut ChunkBuilder, e: &Expr) -> Result<(), String> {
self.compile_expr(b, e)?;
b.emit(Op::CallBuiltin(ops::TRUTHY, 1), 0);
Ok(())
}
fn compile_if(
&mut self,
b: &mut ChunkBuilder,
test: &Expr,
body: &[Stmt],
orelse: &[Stmt],
) -> Result<(), String> {
self.compile_condition(b, test)?;
let jfalse = b.emit(Op::JumpIfFalse(0), 0);
self.compile_stmts(b, body)?;
if orelse.is_empty() {
let end = b.current_pos();
b.patch_jump(jfalse, end);
} else {
let jend = b.emit(Op::Jump(0), 0);
let else_start = b.current_pos();
b.patch_jump(jfalse, else_start);
self.compile_stmts(b, orelse)?;
let end = b.current_pos();
b.patch_jump(jend, end);
}
Ok(())
}
fn compile_while(
&mut self,
b: &mut ChunkBuilder,
test: &Expr,
body: &[Stmt],
orelse: &[Stmt],
) -> Result<(), String> {
let mut versioned: Option<NativeForPlan> = None;
if orelse.is_empty() {
if let Some(plan) = self.try_compile_native_while(b, test, body)? {
if plan.skip_generic.is_none() {
return Ok(());
}
let generic_start = b.current_pos();
for j in &plan.guard_jumps {
b.patch_jump(*j, generic_start);
}
versioned = Some(plan);
}
}
if loop_needs_signal(body) {
return self.compile_while_signal(b, test, body, orelse);
}
let start = b.current_pos();
self.compile_condition(b, test)?;
let jfalse = b.emit(Op::JumpIfFalse(0), 0);
self.loops.push(LoopCtx {
breaks: Vec::new(),
continues: Vec::new(),
signal: false,
});
self.compile_stmts(b, body)?;
b.emit(Op::Jump(start), 0);
let ctx = self.loops.pop().unwrap();
for c in ctx.continues {
b.patch_jump(c, start);
}
let after_cond = b.current_pos();
b.patch_jump(jfalse, after_cond);
if !orelse.is_empty() {
self.compile_stmts(b, orelse)?;
}
let end = b.current_pos();
for br in ctx.breaks {
b.patch_jump(br, end);
}
if let Some(plan) = versioned {
if let Some(j) = plan.skip_generic {
let after_generic = b.current_pos();
b.patch_jump(j, after_generic);
}
}
Ok(())
}
fn try_compile_native_range_for(
&mut self,
b: &mut ChunkBuilder,
target: &Expr,
iter: &Expr,
body: &[Stmt],
) -> Result<Option<NativeForPlan>, String> {
if self.native_slots.is_some() {
return Ok(None);
}
let target_name = match target.unspanned() {
Expr::Name(n) => n.clone(),
_ => return Ok(None),
};
let (start_expr, stop_expr, step) = match native_range_call(iter) {
Some(b) => b,
None => return Ok(None),
};
if self.func_scopes.iter().any(|s| s.contains("range")) {
return Ok(None);
}
let mut loop_vars = vec![target_name.clone()];
let mut reads: Vec<String> = Vec::new();
let mut writes = vec![target_name.clone()];
if let Some(e) = start_expr {
if !native_safe_value(e, &mut reads, IntNames::Only(&[])) {
return Ok(None);
}
}
if !native_safe_value(stop_expr, &mut reads, IntNames::Only(&[])) {
return Ok(None);
}
if !analyze_native_tree(body, &mut loop_vars, &mut reads, &mut writes, IntNames::All) {
return Ok(None);
}
let mut defined = vec![target_name.clone()];
let mut load_names: Vec<String> = Vec::new();
collect_ns_loads(body, &mut defined, &mut load_names);
if !native_writebacks_valid(&writes, &load_names, &defined, &loop_vars, true) {
return Ok(None);
}
let mut slots: HashMap<String, u16> = HashMap::new();
let mut next: u16 = self.fn_slots.values().copied().max().map_or(0, |m| m + 1);
for name in loop_vars.iter().chain(reads.iter()).chain(writes.iter()) {
if !slots.contains_key(name) {
match self.fn_slots.get(name) {
Some(s) => {
slots.insert(name.clone(), *s);
}
None => {
slots.insert(name.clone(), next);
next += 1;
}
}
}
}
let target_slot = slots[&target_name];
let outer_stop_slot = next;
next += 1;
let ns_loads: Vec<(String, u16)> = load_names
.iter()
.filter(|n| !self.fn_slots.contains_key(*n))
.map(|n| (n.clone(), slots[n]))
.collect();
let guard_slots: Vec<u16> = load_names.iter().map(|n| slots[n]).collect();
let mut wb: Vec<(String, u16)> = Vec::new();
for name in writes.iter().filter(|n| !self.fn_slots.contains_key(*n)) {
if !wb.iter().any(|(w, _)| w == name) {
wb.push((name.clone(), slots[name]));
}
}
b.set_int_overflow_deopt(true);
match start_expr {
Some(e) => self.compile_expr(b, e)?,
None => {
b.emit(Op::LoadInt(0), 0);
}
}
b.emit(Op::SetSlot(target_slot), 0);
self.compile_expr(b, stop_expr)?;
b.emit(Op::SetSlot(outer_stop_slot), 0);
self.native_slots = Some(slots);
self.native_next_slot = next;
let mut guard_jumps = Vec::new();
let res = self.emit_loop_core(
b,
NativeLoopRange {
target_slot,
stop_slot: outer_stop_slot,
step,
},
body,
NativeLoopNs {
loads: &ns_loads,
writebacks: &wb,
guard_slots: &guard_slots,
},
&mut guard_jumps,
);
self.native_slots = None;
res?;
let skip_generic = if guard_jumps.is_empty() {
None
} else {
Some(b.emit(Op::Jump(0), 0))
};
Ok(Some(NativeForPlan {
guard_jumps,
skip_generic,
}))
}
fn emit_native_range_loop(
&mut self,
b: &mut ChunkBuilder,
target_name: &str,
start_expr: Option<&Expr>,
stop_expr: &Expr,
step: i64,
body: &[Stmt],
) -> Result<(), String> {
let target_slot = self.native_slots.as_ref().unwrap()[target_name];
let stop_slot = self.native_next_slot;
self.native_next_slot += 1;
match start_expr {
Some(e) => self.compile_expr(b, e)?,
None => {
b.emit(Op::LoadInt(0), 0);
}
}
b.emit(Op::SetSlot(target_slot), 0);
self.compile_expr(b, stop_expr)?;
b.emit(Op::SetSlot(stop_slot), 0);
self.emit_loop_core(
b,
NativeLoopRange {
target_slot,
stop_slot,
step,
},
body,
NativeLoopNs::EMPTY,
&mut Vec::new(),
)
}
fn emit_loop_core(
&mut self,
b: &mut ChunkBuilder,
range: NativeLoopRange,
body: &[Stmt],
ns: NativeLoopNs<'_>,
guard_jumps: &mut Vec<usize>,
) -> Result<(), String> {
let NativeLoopRange {
target_slot,
stop_slot,
step,
} = range;
let NativeLoopNs {
loads: ns_loads,
writebacks: ns_writebacks,
guard_slots,
} = ns;
let cmp = if step > 0 { Op::NumLt } else { Op::NumGt };
b.emit(Op::GetSlot(target_slot), 0);
b.emit(Op::GetSlot(stop_slot), 0);
b.emit(cmp.clone(), 0);
let jskip = b.emit(Op::JumpIfFalse(0), 0);
for (name, slot) in ns_loads {
self.name_const(b, name);
b.emit(Op::CallBuiltin(ops::GETLOCAL, 1), 0);
b.emit(Op::SetSlot(*slot), 0);
}
for slot in guard_slots {
b.emit(Op::GetSlot(*slot), 0);
b.emit(Op::CallBuiltin(ops::IS_INT, 1), 0);
guard_jumps.push(b.emit(Op::JumpIfFalse(0), 0));
}
let top = b.current_pos();
self.emit_native_body(b, body)?;
b.emit(Op::GetSlot(target_slot), 0);
b.emit(Op::LoadInt(step), 0);
b.emit(Op::Add, 0);
b.emit(Op::SetSlot(target_slot), 0);
b.emit(Op::GetSlot(target_slot), 0);
b.emit(Op::GetSlot(stop_slot), 0);
b.emit(cmp, 0);
b.emit(Op::JumpIfTrue(top), 0);
b.emit(Op::GetSlot(target_slot), 0);
b.emit(Op::LoadInt(step), 0);
b.emit(Op::Sub, 0);
b.emit(Op::SetSlot(target_slot), 0);
for (name, slot) in ns_writebacks {
b.emit(Op::GetSlot(*slot), 0); self.name_const(b, name); b.emit(Op::Swap, 0); b.emit(Op::CallBuiltin(ops::SETLOCAL, 2), 0);
b.emit(Op::Pop, 0);
}
let after = b.current_pos();
b.patch_jump(jskip, after);
Ok(())
}
fn emit_native_body(&mut self, b: &mut ChunkBuilder, body: &[Stmt]) -> Result<(), String> {
for stmt in body {
match &stmt.kind {
StmtKind::For {
target,
iter,
body: inner,
..
} => {
let tname = match target.unspanned() {
Expr::Name(n) => n.clone(),
_ => unreachable!("native nest validated"),
};
let (istart, istop, istep) =
native_range_call(iter).expect("native nest validated");
self.emit_native_range_loop(b, &tname, istart, istop, istep, inner)?;
}
StmtKind::While {
test, body: wbody, ..
} => {
self.emit_native_while(b, test, wbody)?;
}
StmtKind::If {
test,
body: tb,
orelse,
} => {
if let Some((target_slot, nop, value)) = self.branchless_accum(tb, orelse) {
b.emit(Op::GetSlot(target_slot), 0);
self.emit_native_cond(b, test)?;
b.emit(Op::LoadInt(1), 0);
b.emit(Op::BitAnd, 0);
self.compile_expr(b, value)?;
b.emit(Op::Mul, 0);
b.emit(nop, 0);
b.emit(Op::SetSlot(target_slot), 0);
} else {
self.emit_native_if(b, test, tb, orelse)?;
}
}
_ => self.compile_stmts(b, std::slice::from_ref(stmt))?,
}
}
Ok(())
}
fn branchless_accum<'a>(
&self,
then_body: &'a [Stmt],
else_body: &[Stmt],
) -> Option<(u16, Op, &'a Expr)> {
if !else_body.is_empty() || then_body.len() != 1 {
return None;
}
if let StmtKind::AugAssign { target, op, value } = &then_body[0].kind {
if let Expr::Name(n) = target.unspanned() {
let nop = match op {
BinOp::Add => Op::Add,
BinOp::Sub => Op::Sub,
_ => return None,
};
let slot = self.native_slots.as_ref()?.get(n).copied()?;
return Some((slot, nop, value));
}
}
None
}
fn emit_native_mod(
&mut self,
b: &mut ChunkBuilder,
dividend: &Expr,
k: i64,
) -> Result<(), String> {
match dividend.unspanned() {
Expr::BinOp(BinOp::Add, sum_l, sum_r) if mul_operands(sum_l).is_some() => {
let (ml, mr) = mul_operands(sum_l).unwrap();
self.compile_expr(b, ml)?;
self.compile_expr(b, mr)?;
self.compile_expr(b, sum_r)?;
b.emit(Op::LoadInt(k), 0);
let idx = b.emit(Op::MulAddModFloor, self.cur_line);
self.record_span(idx);
return Ok(());
}
Expr::BinOp(BinOp::Add, sum_l, sum_r) if mul_operands(sum_r).is_some() => {
let (ml, mr) = mul_operands(sum_r).unwrap();
self.compile_expr(b, ml)?;
self.compile_expr(b, mr)?;
self.compile_expr(b, sum_l)?;
b.emit(Op::LoadInt(k), 0);
let idx = b.emit(Op::MulAddModFloor, self.cur_line);
self.record_span(idx);
return Ok(());
}
Expr::BinOp(BinOp::Mul, l, r) => {
self.compile_expr(b, l)?;
self.compile_expr(b, r)?;
b.emit(Op::LoadInt(k), 0);
let idx = b.emit(Op::MulModFloor, self.cur_line);
self.record_span(idx);
return Ok(());
}
_ => {
self.compile_expr(b, dividend)?;
b.emit(Op::LoadInt(k), 0);
let idx = b.emit(Op::Mod, self.cur_line);
self.record_span(idx);
}
}
b.emit(Op::Dup, 0);
b.emit(Op::LoadInt(0), 0);
b.emit(if k > 0 { Op::NumLt } else { Op::NumGt }, 0);
b.emit(Op::LoadInt(1), 0); b.emit(Op::BitAnd, 0);
b.emit(Op::LoadInt(k), 0);
b.emit(Op::Mul, 0);
b.emit(Op::Add, 0);
Ok(())
}
fn emit_native_cond(&mut self, b: &mut ChunkBuilder, test: &Expr) -> Result<(), String> {
match test.unspanned() {
Expr::Compare(lhs, rest) if rest.len() == 1 && native_cmp_op(rest[0].0).is_some() => {
self.compile_expr(b, lhs)?;
self.compile_expr(b, &rest[0].1)?;
b.emit(native_cmp_op(rest[0].0).unwrap(), 0);
}
other => {
self.compile_expr(b, other)?;
b.emit(Op::LoadInt(0), 0);
b.emit(Op::NumNe, 0);
}
}
Ok(())
}
fn emit_native_if(
&mut self,
b: &mut ChunkBuilder,
test: &Expr,
then_body: &[Stmt],
else_body: &[Stmt],
) -> Result<(), String> {
self.emit_native_cond(b, test)?;
let jelse = b.emit(Op::JumpIfFalse(0), 0);
self.emit_native_body(b, then_body)?;
if else_body.is_empty() {
let end = b.current_pos();
b.patch_jump(jelse, end);
} else {
let jend = b.emit(Op::Jump(0), 0);
let else_start = b.current_pos();
b.patch_jump(jelse, else_start);
self.emit_native_body(b, else_body)?;
let end = b.current_pos();
b.patch_jump(jend, end);
}
Ok(())
}
fn emit_native_while(
&mut self,
b: &mut ChunkBuilder,
test: &Expr,
body: &[Stmt],
) -> Result<(), String> {
self.emit_native_cond(b, test)?;
let jskip = b.emit(Op::JumpIfFalse(0), 0);
let top = b.current_pos();
self.emit_native_body(b, body)?;
self.emit_native_cond(b, test)?;
b.emit(Op::JumpIfTrue(top), 0);
let end = b.current_pos();
b.patch_jump(jskip, end);
Ok(())
}
fn try_compile_native_while(
&mut self,
b: &mut ChunkBuilder,
test: &Expr,
body: &[Stmt],
) -> Result<Option<NativeForPlan>, String> {
if self.native_slots.is_some() || loop_needs_signal(body) {
return Ok(None);
}
if self.func_scopes.iter().any(|s| s.contains("range")) {
return Ok(None);
}
let mut loop_vars: Vec<String> = Vec::new();
let mut reads: Vec<String> = Vec::new();
let mut writes: Vec<String> = Vec::new();
if !native_safe_cond(test, &mut reads, IntNames::All) {
return Ok(None);
}
if !analyze_native_tree(body, &mut loop_vars, &mut reads, &mut writes, IntNames::All) {
return Ok(None);
}
let mut defined: Vec<String> = Vec::new();
let mut load_names: Vec<String> = Vec::new();
reads_needing_load(test, &defined, &mut load_names);
collect_ns_loads(body, &mut defined, &mut load_names);
if !native_writebacks_valid(&writes, &load_names, &defined, &loop_vars, false) {
return Ok(None);
}
let mut slots: HashMap<String, u16> = HashMap::new();
let mut next: u16 = self.fn_slots.values().copied().max().map_or(0, |m| m + 1);
for name in reads.iter().chain(writes.iter()) {
if !slots.contains_key(name) {
match self.fn_slots.get(name) {
Some(s) => {
slots.insert(name.clone(), *s);
}
None => {
slots.insert(name.clone(), next);
next += 1;
}
}
}
}
let ns_loads: Vec<(String, u16)> = load_names
.iter()
.filter(|n| !self.fn_slots.contains_key(*n))
.map(|n| (n.clone(), slots[n]))
.collect();
let _guard_slots: Vec<u16> = load_names.iter().map(|n| slots[n]).collect();
let mut wb: Vec<(String, u16)> = Vec::new();
for name in writes.iter().filter(|n| !self.fn_slots.contains_key(*n)) {
if !wb.iter().any(|(w, _)| w == name) {
wb.push((name.clone(), slots[name]));
}
}
b.set_int_overflow_deopt(true);
self.native_slots = Some(slots);
self.native_next_slot = next;
for (name, slot) in &ns_loads {
self.name_const(b, name);
b.emit(Op::CallBuiltin(ops::GETLOCAL, 1), 0);
b.emit(Op::SetSlot(*slot), 0);
}
let mut guard_jumps = Vec::new();
for (_, slot) in &ns_loads {
b.emit(Op::GetSlot(*slot), 0);
b.emit(Op::CallBuiltin(ops::IS_INT, 1), 0);
guard_jumps.push(b.emit(Op::JumpIfFalse(0), 0));
}
let res = self.emit_native_while(b, test, body);
if res.is_ok() {
for (name, slot) in &wb {
b.emit(Op::GetSlot(*slot), 0);
self.name_const(b, name);
b.emit(Op::Swap, 0);
b.emit(Op::CallBuiltin(ops::SETLOCAL, 2), 0);
b.emit(Op::Pop, 0);
}
}
self.native_slots = None;
res?;
let skip_generic = if guard_jumps.is_empty() {
None
} else {
Some(b.emit(Op::Jump(0), 0))
};
Ok(Some(NativeForPlan {
guard_jumps,
skip_generic,
}))
}
fn bind_loop_target(&mut self, target: &Expr) -> Vec<String> {
let mut added = Vec::new();
let mut mark = |c: &mut Self, n: &str| {
if c.fn_slots.contains_key(n) && c.fn_slots_bound.insert(n.to_string()) {
added.push(n.to_string());
}
};
match target.unspanned() {
Expr::Name(n) => mark(self, n),
Expr::Tuple(xs) | Expr::List(xs) => {
for x in xs {
if let Expr::Name(n) = x.unspanned() {
mark(self, n);
}
}
}
_ => {}
}
added
}
fn unbind_loop_target(&mut self, added: Vec<String>) {
for n in added {
self.fn_slots_bound.remove(&n);
}
}
fn compile_for(
&mut self,
b: &mut ChunkBuilder,
target: &Expr,
iter: &Expr,
body: &[Stmt],
orelse: &[Stmt],
) -> Result<(), String> {
let bound_target = self.bind_loop_target(target);
let r = self.compile_for_inner(b, target, iter, body, orelse);
self.unbind_loop_target(bound_target);
r
}
fn compile_for_inner(
&mut self,
b: &mut ChunkBuilder,
target: &Expr,
iter: &Expr,
body: &[Stmt],
orelse: &[Stmt],
) -> Result<(), String> {
let mut versioned: Option<NativeForPlan> = None;
if orelse.is_empty() && !loop_needs_signal(body) {
if let Some(plan) = self.try_compile_native_range_for(b, target, iter, body)? {
if plan.skip_generic.is_none() {
return Ok(());
}
let generic_start = b.current_pos();
for j in &plan.guard_jumps {
b.patch_jump(*j, generic_start);
}
versioned = Some(plan);
}
}
if loop_needs_signal(body) {
return self.compile_for_signal(b, target, iter, body, orelse);
}
self.compile_expr(b, iter)?;
b.emit(Op::CallBuiltin(ops::GETITER, 1), 0); let start = b.current_pos();
b.emit(Op::CallBuiltin(ops::FORITER, 0), 0); let jdone = b.emit(Op::JumpIfFalse(0), 0); self.compile_assign(b, target)?; self.loops.push(LoopCtx {
breaks: Vec::new(),
continues: Vec::new(),
signal: false,
});
self.compile_stmts(b, body)?;
b.emit(Op::Jump(start), 0);
let ctx = self.loops.pop().unwrap();
for c in ctx.continues {
b.patch_jump(c, start);
}
let done = b.current_pos();
b.patch_jump(jdone, done);
b.emit(Op::Pop, 0); if !orelse.is_empty() {
self.compile_stmts(b, orelse)?;
}
let jafter = b.emit(Op::Jump(0), 0);
let break_target = b.current_pos();
b.emit(Op::Pop, 0);
let end = b.current_pos();
b.patch_jump(jafter, end);
for br in ctx.breaks {
b.patch_jump(br, break_target);
}
if let Some(plan) = versioned {
if let Some(j) = plan.skip_generic {
let after_generic = b.current_pos();
b.patch_jump(j, after_generic);
}
}
Ok(())
}
fn register_loop_body(&mut self, body: &[Stmt]) -> Result<usize, String> {
self.loops.push(LoopCtx {
breaks: Vec::new(),
continues: Vec::new(),
signal: true,
});
let chunk = self.compile_block_chunk(body);
self.loops.pop();
let chunk = chunk?;
let id = self.tries.len();
self.tries.push(TryDef {
body: chunk,
handlers: Vec::new(),
orelse: None,
finalbody: None,
});
Ok(id)
}
fn compile_while_signal(
&mut self,
b: &mut ChunkBuilder,
test: &Expr,
body: &[Stmt],
orelse: &[Stmt],
) -> Result<(), String> {
let id = self.register_loop_body(body)?;
let start = b.current_pos();
self.compile_condition(b, test)?;
let jfalse = b.emit(Op::JumpIfFalse(0), 0);
b.emit(Op::LoadInt(id as i64), 0);
b.emit(Op::CallBuiltin(ops::LOOP_BODY, 1), 0); let jbreak = b.emit(Op::JumpIfTrue(0), 0); b.emit(Op::Jump(start), 0); let after_cond = b.current_pos();
b.patch_jump(jfalse, after_cond);
if !orelse.is_empty() {
self.compile_stmts(b, orelse)?;
}
let end = b.current_pos();
b.patch_jump(jbreak, end);
Ok(())
}
fn compile_for_signal(
&mut self,
b: &mut ChunkBuilder,
target: &Expr,
iter: &Expr,
body: &[Stmt],
orelse: &[Stmt],
) -> Result<(), String> {
let id = self.register_loop_body(body)?;
self.compile_expr(b, iter)?;
b.emit(Op::CallBuiltin(ops::GETITER, 1), 0); let start = b.current_pos();
b.emit(Op::CallBuiltin(ops::FORITER, 0), 0); let jdone = b.emit(Op::JumpIfFalse(0), 0); self.compile_assign(b, target)?; b.emit(Op::LoadInt(id as i64), 0);
b.emit(Op::CallBuiltin(ops::LOOP_BODY, 1), 0); let jbreak = b.emit(Op::JumpIfTrue(0), 0); b.emit(Op::Jump(start), 0); let done = b.current_pos();
b.patch_jump(jdone, done);
b.emit(Op::Pop, 0); if !orelse.is_empty() {
self.compile_stmts(b, orelse)?;
}
let jafter = b.emit(Op::Jump(0), 0);
let break_target = b.current_pos();
b.emit(Op::Pop, 0); let end = b.current_pos();
b.patch_jump(jafter, end);
b.patch_jump(jbreak, break_target);
Ok(())
}
fn compile_async_for(
&mut self,
b: &mut ChunkBuilder,
target: &Expr,
iter: &Expr,
body: &[Stmt],
orelse: &[Stmt],
) -> Result<(), String> {
let ait = format!(".aiter{}", self.tmp);
let run = format!(".arun{}", self.tmp);
let val = format!(".aval{}", self.tmp);
self.tmp += 1;
let mut pre: Vec<Stmt> = Vec::new();
pre.push(
StmtKind::Assign {
targets: vec![Expr::Name(ait.clone())],
value: Expr::Call {
func: Box::new(Expr::Attribute(Box::new(iter.clone()), "__aiter__".into())),
args: vec![],
keywords: vec![],
},
}
.into(),
);
pre.push(
StmtKind::Assign {
targets: vec![Expr::Name(run.clone())],
value: Expr::True,
}
.into(),
);
let anext = Expr::Await(Box::new(Expr::Call {
func: Box::new(Expr::Attribute(
Box::new(Expr::Name(ait)),
"__anext__".into(),
)),
args: vec![],
keywords: vec![],
}));
let try_stmt: Stmt = StmtKind::Try {
body: vec![StmtKind::Assign {
targets: vec![Expr::Name(val.clone())],
value: anext,
}
.into()],
handlers: vec![ExceptHandler {
typ: Some(Expr::Name("StopAsyncIteration".into())),
name: None,
body: vec![StmtKind::Assign {
targets: vec![Expr::Name(run.clone())],
value: Expr::False,
}
.into()],
star: false,
}],
orelse: vec![],
finalbody: vec![],
}
.into();
let mut guarded: Vec<Stmt> = vec![StmtKind::Assign {
targets: vec![target.clone()],
value: Expr::Name(val),
}
.into()];
guarded.extend_from_slice(body);
let loop_body: Vec<Stmt> = vec![
try_stmt,
StmtKind::If {
test: Expr::Name(run.clone()),
body: guarded,
orelse: vec![],
}
.into(),
];
pre.push(
StmtKind::While {
test: Expr::Name(run),
body: loop_body,
orelse: orelse.to_vec(),
}
.into(),
);
self.compile_stmts(b, &pre)
}
fn compile_assert(
&mut self,
b: &mut ChunkBuilder,
test: &Expr,
msg: &Option<Expr>,
) -> Result<(), String> {
self.compile_condition(b, test)?;
let jok = b.emit(Op::JumpIfTrue(0), 0);
match msg {
Some(m) => self.compile_expr(b, m)?,
None => {
b.emit(Op::LoadUndef, 0);
}
}
b.emit(Op::CallBuiltin(ops::ASSERT_FAIL, 1), 0);
let ok = b.current_pos();
b.patch_jump(jok, ok);
Ok(())
}
fn compile_funcdef(
&mut self,
b: &mut ChunkBuilder,
name: &str,
params: &Params,
body: &[Stmt],
decorators: &[Expr],
is_async: bool,
) -> Result<(), String> {
self.fn_depth += 1;
let def_id = self.build_function_ex(name, params, body, is_async, ScopeKind::Function);
self.fn_depth -= 1;
let def_id = def_id?;
self.emit_make_func(b, def_id, params)?;
for d in decorators.iter().rev() {
self.compile_expr(b, d)?; b.emit(Op::Swap, 0); b.emit(Op::CallBuiltin(ops::CALL_VALUE, 2), 0);
}
self.store_name(b, name);
Ok(())
}
fn emit_make_func(
&mut self,
b: &mut ChunkBuilder,
def_id: usize,
params: &Params,
) -> Result<(), String> {
if params.annotations.is_empty() {
b.emit(Op::CallBuiltin(ops::MKDICT, 0), 0);
} else {
let dict_expr = Expr::Dict(
params
.annotations
.iter()
.map(|(name, ann)| (Some(Expr::Str(name.clone())), ann.clone()))
.collect(),
);
let bodystmt = vec![Stmt::from(StmtKind::Return(Some(dict_expr)))];
let empty = Params::default();
self.fn_depth += 1;
let thunk_id = self.build_function("<annotate>", &empty, &bodystmt);
self.fn_depth -= 1;
self.emit_make_func(b, thunk_id?, &empty)?; }
for d in ¶ms.defaults {
self.compile_expr(b, d)?;
}
let mut nkw = 0usize;
for e in params.kwonly_defaults.iter().flatten() {
self.compile_expr(b, e)?;
nkw += 1;
}
b.emit(Op::LoadInt(nkw as i64), 0); b.emit(Op::LoadInt(def_id as i64), 0); let total = 1 + params.defaults.len() + nkw + 2; b.emit(Op::CallBuiltin(ops::MKFUNC, argc(total)?), 0);
Ok(())
}
fn build_function(
&mut self,
name: &str,
params: &Params,
body: &[Stmt],
) -> Result<usize, String> {
self.build_function_ex(name, params, body, false, ScopeKind::Function)
}
fn check_nonlocal_binding(&self, name: &str) -> Result<(), String> {
let n = self.func_scopes.len();
if n == 0 {
return Err("SyntaxError: nonlocal declaration not allowed at module level".into());
}
let bound = self.func_scopes[..n - 1].iter().any(|s| s.contains(name));
if !bound {
return Err(format!(
"SyntaxError: no binding for nonlocal '{name}' found"
));
}
Ok(())
}
fn build_function_ex(
&mut self,
name: &str,
params: &Params,
body: &[Stmt],
is_async: bool,
kind: ScopeKind,
) -> Result<usize, String> {
self.build_function_qn(name, name, params, body, is_async, kind)
}
fn build_function_qn(
&mut self,
name: &str,
qual_name: &str,
params: &Params,
body: &[Stmt],
is_async: bool,
kind: ScopeKind,
) -> Result<usize, String> {
let qualname = format!("{}{}", self.qual_prefix, qual_name);
let saved_prefix = self.qual_prefix.clone();
self.qual_prefix = match kind {
ScopeKind::ClassBody => format!("{qualname}."),
_ => format!("{qualname}.<locals>."),
};
let locals = if kind == ScopeKind::ClassBody {
Vec::new()
} else {
scope_locals(body)
};
let freevars = if kind.is_function() {
scope_freevars(params, body, &self.func_scopes)
} else {
Vec::new()
};
let mut pushed = false;
if kind.is_function() {
let mut bound: HashSet<String> = locals.iter().cloned().collect();
for p in param_names(params) {
bound.insert(p);
}
self.func_scopes.push(bound);
pushed = true;
}
let saved_slots = std::mem::take(&mut self.fn_slots);
let saved_bound = std::mem::take(&mut self.fn_slots_bound);
let is_gen_or_async = is_async || body_has_yield(body);
if kind.is_function() && !is_gen_or_async && fn_slots_allowed(body) {
let mut next: u16 = 0;
let mut table: HashMap<String, u16> = HashMap::new();
for p in param_names(params) {
table.entry(p).or_insert_with(|| {
let s = next;
next += 1;
s
});
}
for l in &locals {
if freevars.contains(l) {
continue;
}
table.entry(l.clone()).or_insert_with(|| {
let s = next;
next += 1;
s
});
}
let mut proven = provably_bound_locals(body);
for p in param_names(params) {
proven.insert(p);
}
self.fn_slots = table;
self.fn_slots_bound = proven;
}
let mut fb = ChunkBuilder::new();
self.begin_chunk();
if !self.fn_slots.is_empty() {
let params_set: HashSet<String> = param_names(params).into_iter().collect();
let mut entries: Vec<(&String, &u16)> = self.fn_slots.iter().collect();
entries.sort_by_key(|(_, s)| **s);
let proven = self.fn_slots_bound.clone();
for (name, slot) in entries {
if params_set.contains(name) {
self.name_const(&mut fb, name);
fb.emit(Op::CallBuiltin(ops::GETLOCAL, 1), 0);
} else if proven.contains(name) {
continue;
} else {
fb.emit(Op::CallBuiltin(ops::UNBOUND, 0), 0);
}
fb.emit(Op::SetSlot(*slot), 0);
}
}
let saved_icb = self.in_class_body;
let saved_fs = self.def_depth;
self.def_depth = match kind {
ScopeKind::ClassBody => 0,
_ => self.def_depth + 1,
};
let saved_aw = self.await_scope;
let saved_inc = self.in_comprehension;
self.await_scope = match kind {
ScopeKind::ClassBody => AwaitScope::Module,
ScopeKind::Comprehension => saved_aw,
ScopeKind::Function if is_async => AwaitScope::Async,
ScopeKind::Function => AwaitScope::Sync,
};
self.in_comprehension = kind == ScopeKind::Comprehension;
self.in_class_body = kind == ScopeKind::ClassBody;
if self.in_class_body && body.iter().any(is_ann_assign) {
fb.emit(Op::CallBuiltin(ops::MKDICT, 0), 0);
self.name_const(&mut fb, "__annotations__");
fb.emit(Op::Swap, 0);
fb.emit(Op::CallBuiltin(ops::SETLOCAL, 2), 0);
fb.emit(Op::Pop, 0);
}
let compiled = self.compile_stmts(&mut fb, body);
self.fn_slots = saved_slots;
self.fn_slots_bound = saved_bound;
self.in_class_body = saved_icb;
self.def_depth = saved_fs;
self.await_scope = saved_aw;
self.in_comprehension = saved_inc;
if pushed {
self.func_scopes.pop();
}
self.qual_prefix = saved_prefix;
compiled?;
let is_generator = body_has_yield(body);
let def = FuncDef {
name: name.to_string(),
qualname,
params: params.names.clone(),
posonly: params.posonly,
ndefaults: params.defaults.len(),
star: params.star.clone(),
kwonly: params.kwonly.clone(),
kwonly_required: params.kwonly_defaults.iter().map(|d| d.is_none()).collect(),
kwargs: params.kwargs.clone(),
chunk: self.finish_chunk(fb),
locals,
is_generator,
is_async,
doc: docstring(body),
freevars,
};
self.functions.push((name.to_string(), def));
Ok(self.functions.len() - 1)
}
fn compile_classdef(
&mut self,
b: &mut ChunkBuilder,
name: &str,
bases: &[Expr],
keywords: &[Keyword],
body: &[Stmt],
decorators: &[Expr],
) -> Result<(), String> {
let empty = Params::default();
self.fn_depth += 1;
let def_id = self.build_function_qn(
&format!("<class {name}>"),
name,
&empty,
body,
false,
ScopeKind::ClassBody,
);
self.fn_depth -= 1;
let def_id = def_id?;
match keywords
.iter()
.find(|k| k.name.as_deref() == Some("metaclass"))
{
Some(k) => self.compile_expr(b, &k.value)?,
None => {
b.emit(Op::LoadUndef, 0);
}
}
for base in bases {
self.compile_expr(b, base)?;
}
b.emit(Op::CallBuiltin(ops::MKLIST, argc(bases.len())?), 0); self.name_const(b, name); self.emit_make_func(b, def_id, &empty)?; let ckw: Vec<&Keyword> = keywords
.iter()
.filter(|k| k.name.is_some() && k.name.as_deref() != Some("metaclass"))
.collect();
for kw in &ckw {
self.strlit(b, kw.name.as_ref().unwrap());
self.compile_expr(b, &kw.value)?;
}
b.emit(Op::CallBuiltin(ops::MKDICT, argc(ckw.len() * 2)?), 0); b.emit(Op::CallBuiltin(ops::BUILD_CLASS, 5), 0); for d in decorators.iter().rev() {
self.compile_expr(b, d)?;
b.emit(Op::Swap, 0);
b.emit(Op::CallBuiltin(ops::CALL_VALUE, 2), 0);
}
self.store_name(b, name);
Ok(())
}
fn compile_try(
&mut self,
b: &mut ChunkBuilder,
body: &[Stmt],
handlers: &[ExceptHandler],
orelse: &[Stmt],
finalbody: &[Stmt],
) -> Result<(), String> {
if !finalbody.is_empty() {
collect_finally_escapes(finalbody, &mut self.warnings);
}
check_except_star(handlers)?;
let body_chunk = self.compile_block_chunk(body)?;
let mut hs = Vec::new();
for h in handlers {
let type_chunk = match &h.typ {
Some(t) => Some(self.compile_expr_chunk(t)?),
None => None,
};
let hbody = self.compile_block_chunk(&h.body)?;
hs.push(HandlerDef {
typ: type_chunk,
name: h.name.clone(),
body: hbody,
star: h.star,
});
}
let else_chunk = if orelse.is_empty() {
None
} else {
Some(self.compile_block_chunk(orelse)?)
};
let final_chunk = if finalbody.is_empty() {
None
} else {
Some(self.compile_block_chunk(finalbody)?)
};
let id = self.tries.len();
self.tries.push(TryDef {
body: body_chunk,
handlers: hs,
orelse: else_chunk,
finalbody: final_chunk,
});
b.emit(Op::LoadInt(id as i64), 0);
b.emit(Op::CallBuiltin(ops::TRY, 1), 0);
b.emit(Op::Pop, 0);
Ok(())
}
fn compile_block_chunk(&mut self, stmts: &[Stmt]) -> Result<Chunk, String> {
let mut cb = ChunkBuilder::new();
self.begin_chunk();
self.compile_stmts(&mut cb, stmts)?;
Ok(self.finish_chunk(cb))
}
fn compile_expr_chunk(&mut self, e: &Expr) -> Result<Chunk, String> {
let mut cb = ChunkBuilder::new();
self.begin_chunk();
self.compile_expr(&mut cb, e)?;
Ok(self.finish_chunk(cb))
}
fn compile_with(
&mut self,
b: &mut ChunkBuilder,
items: &[WithItem],
body: &[Stmt],
is_async: bool,
) -> Result<(), String> {
let mut cur: Vec<Stmt> = body.to_vec();
for it in items.iter().rev() {
cur = self.desugar_with_single(it, cur, is_async);
}
self.compile_stmts(b, &cur)
}
fn desugar_with_single(&mut self, it: &WithItem, body: Vec<Stmt>, is_async: bool) -> Vec<Stmt> {
let (enter_m, exit_m) = if is_async {
(".__aenter__", "__aexit__")
} else {
(".__enter__", "__exit__")
};
let awaited = |e: Expr| -> Expr {
if is_async {
Expr::Await(Box::new(e))
} else {
e
}
};
let ctx = format!(".with{}", self.tmp);
self.tmp += 1;
let hit = format!(".withhit{}", self.tmp);
self.tmp += 1;
let exc = format!(".withexc{}", self.tmp);
self.tmp += 1;
let call = |recv: &str, method: &str, args: Vec<Expr>| -> Expr {
Expr::Call {
func: Box::new(Expr::Attribute(
Box::new(Expr::Name(recv.into())),
method.into(),
)),
args,
keywords: vec![],
}
};
let mut out: Vec<Stmt> = Vec::new();
out.push(
StmtKind::Assign {
targets: vec![Expr::Name(ctx.clone())],
value: it.context.clone(),
}
.into(),
);
let enter = awaited(call(&ctx, enter_m, vec![]));
match &it.vars {
Some(v) => out.push(
StmtKind::Assign {
targets: vec![v.clone()],
value: enter,
}
.into(),
),
None => out.push(StmtKind::Expr(enter).into()),
}
out.push(
StmtKind::Assign {
targets: vec![Expr::Name(hit.clone())],
value: Expr::False,
}
.into(),
);
let exit_exc = awaited(call(
&ctx,
exit_m,
vec![
Expr::Call {
func: Box::new(Expr::Name("type".into())),
args: vec![Expr::Name(exc.clone())],
keywords: vec![],
},
Expr::Name(exc.clone()),
Expr::None,
],
));
let handler_body: Vec<Stmt> = vec![
StmtKind::Assign {
targets: vec![Expr::Name(hit.clone())],
value: Expr::True,
}
.into(),
StmtKind::If {
test: Expr::UnaryOp(UnOp::Not, Box::new(exit_exc)),
body: vec![StmtKind::Raise {
exc: None,
cause: None,
}
.into()],
orelse: vec![],
}
.into(),
];
let exit_none = awaited(call(&ctx, exit_m, vec![Expr::None, Expr::None, Expr::None]));
let finalbody: Vec<Stmt> = vec![StmtKind::If {
test: Expr::UnaryOp(UnOp::Not, Box::new(Expr::Name(hit.clone()))),
body: vec![StmtKind::Expr(exit_none).into()],
orelse: vec![],
}
.into()];
out.push(
StmtKind::Try {
body,
handlers: vec![ExceptHandler {
typ: Some(Expr::Name("BaseException".into())),
name: Some(exc),
body: handler_body,
star: false,
}],
orelse: vec![],
finalbody,
}
.into(),
);
out
}
fn compile_expr(&mut self, b: &mut ChunkBuilder, e: &Expr) -> Result<(), String> {
if let Expr::Spanned(inner, sp) = e {
let mut sp = *sp;
if std::mem::take(&mut self.suppress_hint)
&& matches!(inner.unspanned(), Expr::Call { .. })
{
sp.suppress = true;
}
let prev = std::mem::replace(&mut self.node_span, sp);
let r = self.compile_expr(b, inner);
self.node_span = prev;
return r;
}
match e {
Expr::Spanned(_, _) => unreachable!("peeled above"),
Expr::None => {
b.emit(Op::LoadUndef, 0);
}
Expr::True => {
b.emit(Op::LoadTrue, 0);
}
Expr::False => {
b.emit(Op::LoadFalse, 0);
}
Expr::Int(n) => {
b.emit(Op::LoadInt(*n), 0);
}
Expr::Float(f) => {
b.emit(Op::LoadFloat(*f), 0);
}
Expr::BigInt(s) => {
let hex = num_bigint::BigInt::parse_bytes(s.as_bytes(), 10)
.map_or_else(|| s.clone(), |n| n.to_str_radix(16));
self.name_const(b, "int");
self.strlit(b, &hex);
b.emit(Op::LoadInt(16), 0);
b.emit(Op::CallBuiltin(ops::CALL, 3), 0);
}
Expr::Complex(f) => {
self.name_const(b, "complex");
b.emit(Op::LoadFloat(0.0), 0);
b.emit(Op::LoadFloat(*f), 0);
b.emit(Op::CallBuiltin(ops::CALL, 3), 0);
}
Expr::Ellipsis => {
b.emit(Op::CallBuiltin(ops::ELLIPSIS, 0), 0);
}
Expr::Str(s) => self.strlit(b, s),
Expr::Bytes(bytes) => {
let packed: String = bytes.iter().map(|&byte| byte as char).collect();
let k = b.add_constant(Value::str(&packed));
b.emit(Op::LoadConst(k), 0);
b.emit(Op::CallBuiltin(ops::MKBYTES, 1), 0);
}
Expr::FString(parts) => self.compile_fstring(b, parts)?,
Expr::TString(parts) => self.compile_tstring(b, parts)?,
Expr::Name(n) => {
if let Some(slot) = self.slot_of(n) {
b.emit(Op::GetSlot(slot), self.cur_line);
if self.fn_slots.contains_key(n) && !self.fn_slots_bound.contains(n) {
self.name_const(b, n);
b.emit(Op::Swap, 0);
b.emit(Op::CallBuiltin(ops::CHECK_BOUND, 2), self.cur_line);
}
} else {
self.name_const(b, n);
let idx = b.emit(Op::CallBuiltin(ops::GETLOCAL, 1), self.cur_line);
self.record_span(idx);
}
}
Expr::List(items) => {
if items.iter().any(|e| matches!(e, Expr::Starred(_))) {
self.compile_arg_spread(b, items)?;
} else {
self.build_chunked(
b,
ops::MKLIST,
ops::EXTEND_LIST,
items.len(),
1,
|c, b, i| c.compile_expr(b, &items[i]),
)?;
}
}
Expr::Tuple(items) => {
if items.iter().any(|e| matches!(e, Expr::Starred(_))) {
self.name_const(b, "tuple");
self.compile_arg_spread(b, items)?;
b.emit(Op::CallBuiltin(ops::CALL, 2), 0);
} else {
self.build_chunked(
b,
ops::MKTUPLE,
ops::EXTEND_TUPLE,
items.len(),
1,
|c, b, i| c.compile_expr(b, &items[i]),
)?;
}
}
Expr::Set(items) => {
if items.iter().any(|e| matches!(e, Expr::Starred(_))) {
self.name_const(b, "set");
self.compile_arg_spread(b, items)?;
b.emit(Op::CallBuiltin(ops::CALL, 2), 0);
} else {
self.build_chunked(
b,
ops::MKSET,
ops::EXTEND_SET,
items.len(),
1,
|c, b, i| c.compile_expr(b, &items[i]),
)?;
}
}
Expr::Dict(pairs) => {
if pairs.iter().any(|(k, _)| k.is_none()) {
for (k, v) in pairs {
match k {
Some(k) => {
b.emit(Op::LoadInt(0), 0);
self.compile_expr(b, k)?;
self.compile_expr(b, v)?;
}
None => {
b.emit(Op::LoadInt(1), 0);
self.compile_expr(b, v)?;
b.emit(Op::LoadUndef, 0);
}
}
}
let idx = b.emit(
Op::CallBuiltin(ops::MKDICT_EX, argc(pairs.len() * 3)?),
self.cur_line,
);
self.record_span(idx);
} else {
self.build_chunked(
b,
ops::MKDICT,
ops::EXTEND_DICT,
pairs.len(),
2,
|c, b, i| {
let (k, v) = &pairs[i];
c.compile_expr(b, k.as_ref().expect("plain dict key"))?;
c.compile_expr(b, v)
},
)?;
}
}
Expr::Starred(inner) => self.compile_expr(b, inner)?,
Expr::BoolOp(op, values) => self.compile_boolop(b, *op, values)?,
Expr::UnaryOp(op, e) => self.compile_unaryop(b, *op, e)?,
Expr::BinOp(op, l, r) => self.compile_binop(b, *op, l, r)?,
Expr::Compare(left, ops) => self.compile_compare(b, left, ops)?,
Expr::IfExp { test, body, orelse } => {
self.compile_condition(b, test)?;
let jf = b.emit(Op::JumpIfFalse(0), 0);
self.compile_expr(b, body)?;
let je = b.emit(Op::Jump(0), 0);
let els = b.current_pos();
b.patch_jump(jf, els);
self.compile_expr(b, orelse)?;
let end = b.current_pos();
b.patch_jump(je, end);
}
Expr::Call {
func,
args,
keywords,
} => self.compile_call(b, func, args, keywords)?,
Expr::Attribute(recv, attr) => {
self.compile_expr(b, recv)?;
self.name_const(b, attr);
let idx = b.emit(Op::CallBuiltin(ops::GETATTR, 2), self.cur_line);
self.record_span(idx);
}
Expr::Subscript(recv, idx) => {
if let (Some(ct), Some(it)) = (literal_container_type(recv), bad_index_type(idx)) {
self.warnings.push((
self.cur_line,
format!(
"{ct} indices must be integers or slices, not {it}; \
perhaps you missed a comma?"
),
));
}
self.compile_expr(b, recv)?;
self.compile_subscript_index(b, idx)?;
let op_idx = b.emit(Op::CallBuiltin(ops::GETITEM, 2), self.cur_line);
self.record_span(op_idx);
}
Expr::Slice { lo, hi, step } => {
self.compile_opt(b, lo)?;
self.compile_opt(b, hi)?;
self.compile_opt(b, step)?;
b.emit(Op::CallBuiltin(ops::MKSLICE, 3), 0);
}
Expr::Lambda { params, body } => {
let bodystmt = vec![Stmt::from(StmtKind::Return(Some((**body).clone())))];
self.fn_depth += 1;
let def_id = self.build_function("<lambda>", params, &bodystmt);
self.fn_depth -= 1;
let def_id = def_id?;
self.emit_make_func(b, def_id, params)?;
}
Expr::ListComp(elt, comps) => {
check_comprehension_walrus(&[elt], comps, &[])?;
self.compile_comprehension(b, CompKind::List, elt, None, comps)?
}
Expr::SetComp(elt, comps) => {
check_comprehension_walrus(&[elt], comps, &[])?;
self.compile_comprehension(b, CompKind::Set, elt, None, comps)?
}
Expr::GenExp(elt, comps) => {
check_comprehension_walrus(&[elt], comps, &[])?;
self.compile_genexp(b, elt, comps)?
}
Expr::DictComp(k, v, comps) => {
check_comprehension_walrus(&[k, v], comps, &[])?;
self.compile_comprehension(b, CompKind::Dict, k, Some(v), comps)?
}
Expr::NamedExpr(target, value) => {
self.compile_expr(b, value)?;
b.emit(Op::Dup, 0);
self.compile_assign(b, target)?;
}
Expr::Yield(val) => {
if self.def_depth == 0 {
return Err("SyntaxError: 'yield' outside function".to_string());
}
match val {
Some(e) => self.compile_expr(b, e)?,
None => {
b.emit(Op::LoadUndef, 0);
}
}
b.emit(Op::CallBuiltin(ops::YIELDV, 1), 0);
}
Expr::YieldFrom(inner) => {
if self.def_depth == 0 {
return Err("SyntaxError: 'yield from' outside function".to_string());
}
self.compile_yield_from(b, inner)?;
}
Expr::Await(inner) => {
match self.await_scope {
AwaitScope::Async => {}
_ if self.in_comprehension => {
return Err(
"SyntaxError: asynchronous comprehension outside of an asynchronous function"
.to_string(),
)
}
AwaitScope::Module => {
return Err("SyntaxError: 'await' outside function".to_string())
}
AwaitScope::Sync => {
return Err("SyntaxError: 'await' outside async function".to_string())
}
}
self.compile_expr(b, inner)?;
b.emit(Op::CallBuiltin(ops::AWAIT, 1), 0);
}
}
Ok(())
}
fn compile_yield_from(&mut self, b: &mut ChunkBuilder, iter: &Expr) -> Result<(), String> {
self.compile_expr(b, iter)?;
b.emit(Op::CallBuiltin(ops::YIELD_FROM, 1), 0); Ok(())
}
fn compile_opt(&mut self, b: &mut ChunkBuilder, e: &Option<Box<Expr>>) -> Result<(), String> {
match e {
Some(e) => self.compile_expr(b, e)?,
None => {
b.emit(Op::LoadUndef, 0);
}
}
Ok(())
}
fn compile_seq(&mut self, b: &mut ChunkBuilder, items: &[Expr]) -> Result<(), String> {
for it in items {
self.compile_expr(b, it)?;
}
Ok(())
}
fn build_chunked(
&mut self,
b: &mut ChunkBuilder,
mk: u16,
extend: u16,
count: usize,
slots: usize,
mut emit: impl FnMut(&mut Self, &mut ChunkBuilder, usize) -> Result<(), String>,
) -> Result<(), String> {
let mk_cap = 255 / slots;
let ext_cap = (255 - 1) / slots;
let line = self.cur_line;
if count <= mk_cap {
for i in 0..count {
emit(self, b, i)?;
}
let idx = b.emit(Op::CallBuiltin(mk, (count * slots) as u8), line);
self.record_span(idx);
return Ok(());
}
for i in 0..mk_cap {
emit(self, b, i)?;
}
let idx = b.emit(Op::CallBuiltin(mk, (mk_cap * slots) as u8), line);
self.record_span(idx);
let mut i = mk_cap;
while i < count {
let n = ext_cap.min(count - i);
for j in 0..n {
emit(self, b, i + j)?;
}
let idx = b.emit(Op::CallBuiltin(extend, (1 + n * slots) as u8), line);
self.record_span(idx);
i += n;
}
Ok(())
}
fn compile_subscript_index(&mut self, b: &mut ChunkBuilder, idx: &Expr) -> Result<(), String> {
self.compile_expr(b, idx)
}
fn compile_fstring(&mut self, b: &mut ChunkBuilder, parts: &[FStrPart]) -> Result<(), String> {
self.build_chunked(b, ops::MKSTR, ops::EXTEND_STR, parts.len(), 1, |c, b, i| {
match &parts[i] {
FStrPart::Lit(s) => {
let k = b.add_constant(Value::str(s));
b.emit(Op::LoadConst(k), 0);
}
FStrPart::Expr {
expr, conv, spec, ..
} => {
c.compile_expr(b, expr)?;
let conv_i = match conv {
Some('s') => 1,
Some('r') => 2,
Some('a') => 3,
_ => 0,
};
b.emit(Op::LoadInt(conv_i), 0);
c.compile_fstring_spec(b, spec)?;
b.emit(Op::CallBuiltin(ops::FORMAT, 3), 0);
}
}
Ok(())
})
}
fn compile_tstring(&mut self, b: &mut ChunkBuilder, parts: &[FStrPart]) -> Result<(), String> {
self.build_chunked(
b,
ops::MKLIST,
ops::EXTEND_LIST,
parts.len(),
1,
|c, b, i| {
match &parts[i] {
FStrPart::Lit(s) => {
let k = b.add_constant(Value::str(s));
b.emit(Op::LoadConst(k), 0);
}
FStrPart::Expr {
expr,
src,
conv,
spec,
} => {
c.compile_expr(b, expr)?;
let k = b.add_constant(Value::str(src));
b.emit(Op::LoadConst(k), 0);
let conv_i = match conv {
Some('s') => 1,
Some('r') => 2,
Some('a') => 3,
_ => 0,
};
b.emit(Op::LoadInt(conv_i), 0);
c.compile_fstring_spec(b, spec)?;
b.emit(Op::CallBuiltin(ops::INTERPOLATION, 4), 0);
}
}
Ok(())
},
)?;
b.emit(Op::CallBuiltin(ops::TEMPLATE, 1), 0);
Ok(())
}
fn compile_fstring_spec(
&mut self,
b: &mut ChunkBuilder,
spec: &[FStrPart],
) -> Result<(), String> {
if spec.iter().any(|p| matches!(p, FStrPart::Expr { .. })) {
return self.compile_fstring(b, spec);
}
let mut s = String::new();
for p in spec {
if let FStrPart::Lit(l) = p {
s.push_str(l);
}
}
let k = b.add_constant(Value::str(s));
b.emit(Op::LoadConst(k), 0);
Ok(())
}
fn compile_boolop(
&mut self,
b: &mut ChunkBuilder,
op: BoolOp,
values: &[Expr],
) -> Result<(), String> {
self.compile_expr(b, &values[0])?;
let mut ends = Vec::new();
for v in &values[1..] {
b.emit(Op::Dup, 0);
b.emit(Op::CallBuiltin(ops::TRUTHY, 1), 0);
let jump = match op {
BoolOp::And => b.emit(Op::JumpIfFalse(0), 0),
BoolOp::Or => b.emit(Op::JumpIfTrue(0), 0),
};
ends.push(jump);
b.emit(Op::Pop, 0);
self.compile_expr(b, v)?;
}
let end = b.current_pos();
for j in ends {
b.patch_jump(j, end);
}
Ok(())
}
fn compile_unaryop(&mut self, b: &mut ChunkBuilder, op: UnOp, e: &Expr) -> Result<(), String> {
match op {
UnOp::Neg => {
self.compile_expr(b, e)?;
let idx = b.emit(Op::Negate, 0);
self.record_span(idx);
}
UnOp::Not => {
self.compile_condition(b, e)?;
b.emit(Op::LogNot, 0);
}
UnOp::Invert => {
b.emit(Op::LoadInt(unop::INVERT), 0);
self.compile_expr(b, e)?;
let idx = b.emit(Op::CallBuiltin(ops::UNARY, 2), 0);
self.record_span(idx);
}
UnOp::Pos => {
b.emit(Op::LoadInt(unop::POS), 0);
self.compile_expr(b, e)?;
let idx = b.emit(Op::CallBuiltin(ops::UNARY, 2), 0);
self.record_span(idx);
}
}
Ok(())
}
fn compile_binop(
&mut self,
b: &mut ChunkBuilder,
op: BinOp,
l: &Expr,
r: &Expr,
) -> Result<(), String> {
if matches!(op, BinOp::Mod) && self.native_slots.is_some() {
if let Some(k) = native_mod_divisor(r) {
self.emit_native_mod(b, l, k)?;
return Ok(());
}
}
let tag = match op {
BinOp::Add => {
self.compile_expr(b, l)?;
self.compile_expr(b, r)?;
let idx = b.emit(Op::Add, self.cur_line);
self.record_span(idx);
return Ok(());
}
BinOp::Sub => {
self.compile_expr(b, l)?;
self.compile_expr(b, r)?;
let idx = b.emit(Op::Sub, self.cur_line);
self.record_span(idx);
return Ok(());
}
BinOp::Mul => {
self.compile_expr(b, l)?;
self.compile_expr(b, r)?;
let idx = b.emit(Op::Mul, self.cur_line);
self.record_span(idx);
return Ok(());
}
BinOp::Div => bop::DIV,
BinOp::FloorDiv => bop::FLOORDIV,
BinOp::Mod => bop::MOD,
BinOp::Pow => bop::POW,
BinOp::MatMul => bop::MATMUL,
BinOp::BitAnd => bop::BITAND,
BinOp::BitOr => bop::BITOR,
BinOp::BitXor => bop::BITXOR,
BinOp::Shl => bop::SHL,
BinOp::Shr => bop::SHR,
};
b.emit(Op::LoadInt(tag), 0);
self.compile_expr(b, l)?;
self.compile_expr(b, r)?;
let idx = b.emit(Op::CallBuiltin(ops::BINOP, 3), self.cur_line);
self.record_span(idx);
Ok(())
}
fn compile_compare(
&mut self,
b: &mut ChunkBuilder,
left: &Expr,
ops_list: &[(CmpOp, Expr)],
) -> Result<(), String> {
if ops_list.len() == 1 {
self.emit_single_compare(b, left, ops_list[0].0, &ops_list[0].1)?;
return Ok(());
}
let n = ops_list.len();
let mut conj: Vec<Expr> = Vec::new();
let mut prev = left.clone();
for (i, (op, rhs)) in ops_list.iter().enumerate() {
if i + 1 < n {
let t = format!(".cmp{}", self.tmp);
self.tmp += 1;
let walrus =
Expr::NamedExpr(Box::new(Expr::Name(t.clone())), Box::new(rhs.clone()));
conj.push(Expr::Compare(Box::new(prev), vec![(*op, walrus)]));
prev = Expr::Name(t);
} else {
conj.push(Expr::Compare(Box::new(prev), vec![(*op, rhs.clone())]));
prev = rhs.clone();
}
}
let _ = prev;
self.compile_boolop(b, BoolOp::And, &conj)
}
fn emit_single_compare(
&mut self,
b: &mut ChunkBuilder,
left: &Expr,
op: CmpOp,
rhs: &Expr,
) -> Result<(), String> {
match op {
CmpOp::Eq => {
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
let idx = b.emit(Op::NumEq, self.cur_line);
self.record_span(idx);
}
CmpOp::Ne => {
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
let idx = b.emit(Op::NumNe, self.cur_line);
self.record_span(idx);
}
CmpOp::Lt => {
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
let idx = b.emit(Op::NumLt, self.cur_line);
self.record_span(idx);
}
CmpOp::Le => {
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
let idx = b.emit(Op::NumLe, self.cur_line);
self.record_span(idx);
}
CmpOp::Gt => {
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
let idx = b.emit(Op::NumGt, self.cur_line);
self.record_span(idx);
}
CmpOp::Ge => {
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
let idx = b.emit(Op::NumGe, self.cur_line);
self.record_span(idx);
}
CmpOp::Is => {
if let Some(t) = const_literal_type(left).or_else(|| const_literal_type(rhs)) {
self.warnings.push((
self.cur_line,
format!("\"is\" with '{t}' literal. Did you mean \"==\"?"),
));
}
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
b.emit(Op::CallBuiltin(ops::IS, 2), 0);
}
CmpOp::IsNot => {
if let Some(t) = const_literal_type(left).or_else(|| const_literal_type(rhs)) {
self.warnings.push((
self.cur_line,
format!("\"is not\" with '{t}' literal. Did you mean \"!=\"?"),
));
}
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
b.emit(Op::CallBuiltin(ops::IS, 2), 0);
b.emit(Op::LogNot, 0);
}
CmpOp::In => {
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
b.emit(Op::CallBuiltin(ops::CONTAINS, 2), 0);
}
CmpOp::NotIn => {
self.compile_expr(b, left)?;
self.compile_expr(b, rhs)?;
b.emit(Op::CallBuiltin(ops::CONTAINS, 2), 0);
b.emit(Op::LogNot, 0);
}
}
Ok(())
}
fn compile_call(
&mut self,
b: &mut ChunkBuilder,
func: &Expr,
args: &[Expr],
keywords: &[Keyword],
) -> Result<(), String> {
if let Some(t) = compile_time_type(func) {
self.warnings.push((
self.cur_line,
format!("'{t}' object is not callable; perhaps you missed a comma?"),
));
}
let has_star = args.iter().any(|a| matches!(a, Expr::Starred(_)));
let has_kwsplat = keywords.iter().any(|k| k.name.is_none());
if has_star || has_kwsplat {
return self.compile_call_ex(b, func, args, keywords);
}
let named: Vec<&Keyword> = keywords.iter().collect();
let build_kw = |c: &mut Self, b: &mut ChunkBuilder| -> Result<(), String> {
for kw in &named {
let n = kw.name.as_ref().unwrap();
c.strlit(b, n);
c.compile_expr(b, &kw.value)?;
}
b.emit(Op::CallBuiltin(ops::MKDICT, argc(named.len() * 2)?), 0);
Ok(())
};
let span_idx;
match func.unspanned() {
Expr::Attribute(recv, attr) => {
self.compile_expr(b, recv)?;
self.name_const(b, attr);
self.compile_seq(b, args)?;
if named.is_empty() {
span_idx = b.emit(
Op::CallBuiltin(ops::CALL_METHOD, argc(2 + args.len())?),
self.cur_line,
);
} else {
build_kw(self, b)?;
span_idx = b.emit(
Op::CallBuiltin(ops::CALL_METHOD_KW, argc(3 + args.len())?),
self.cur_line,
);
}
}
_ => {
self.compile_expr(b, func)?;
self.compile_seq(b, args)?;
if named.is_empty() {
span_idx = b.emit(
Op::CallBuiltin(ops::CALL_VALUE, argc(1 + args.len())?),
self.cur_line,
);
} else {
build_kw(self, b)?;
span_idx = b.emit(
Op::CallBuiltin(ops::CALL_VALUE_KW, argc(2 + args.len())?),
self.cur_line,
);
}
}
}
self.record_span(span_idx);
Ok(())
}
fn compile_arg_spread(&mut self, b: &mut ChunkBuilder, items: &[Expr]) -> Result<(), String> {
for it in items {
match it {
Expr::Starred(inner) => {
b.emit(Op::LoadInt(1), 0);
self.compile_expr(b, inner)?;
}
_ => {
b.emit(Op::LoadInt(0), 0);
self.compile_expr(b, it)?;
}
}
}
b.emit(Op::CallBuiltin(ops::BUILD_ARGS, argc(items.len() * 2)?), 0);
Ok(())
}
fn compile_kw_spread(&mut self, b: &mut ChunkBuilder, kws: &[Keyword]) -> Result<(), String> {
for kw in kws {
match &kw.name {
Some(n) => self.strlit(b, n),
None => {
b.emit(Op::LoadUndef, 0);
}
}
self.compile_expr(b, &kw.value)?;
}
b.emit(Op::CallBuiltin(ops::BUILD_KWARGS, argc(kws.len() * 2)?), 0);
Ok(())
}
fn compile_call_ex(
&mut self,
b: &mut ChunkBuilder,
func: &Expr,
args: &[Expr],
keywords: &[Keyword],
) -> Result<(), String> {
match func.unspanned() {
Expr::Attribute(recv, attr) => {
self.compile_expr(b, recv)?;
self.name_const(b, attr);
self.compile_arg_spread(b, args)?;
self.compile_kw_spread(b, keywords)?;
b.emit(Op::CallBuiltin(ops::CALL_METHOD_EX, 4), self.cur_line);
}
Expr::Name(n) if self.slot_of(n).is_some() => {
self.compile_expr(b, func)?;
self.compile_arg_spread(b, args)?;
self.compile_kw_spread(b, keywords)?;
b.emit(Op::CallBuiltin(ops::CALL_VALUE_EX, 3), self.cur_line);
}
Expr::Name(n) => {
self.name_const(b, n);
self.compile_arg_spread(b, args)?;
self.compile_kw_spread(b, keywords)?;
b.emit(Op::CallBuiltin(ops::CALL_EX, 3), self.cur_line);
}
_ => {
self.compile_expr(b, func)?;
self.compile_arg_spread(b, args)?;
self.compile_kw_spread(b, keywords)?;
b.emit(Op::CallBuiltin(ops::CALL_VALUE_EX, 3), self.cur_line);
}
}
Ok(())
}
fn compile_comprehension(
&mut self,
b: &mut ChunkBuilder,
kind: CompKind,
elt: &Expr,
val: Option<&Expr>,
comps: &[Comprehension],
) -> Result<(), String> {
let acc = ".result";
let empty = match kind {
CompKind::List => Expr::List(vec![]),
CompKind::Set => Expr::Set(vec![]),
CompKind::Dict => Expr::Dict(vec![]),
};
let add_stmt: Stmt = match kind {
CompKind::List => StmtKind::Expr(Expr::Call {
func: Box::new(Expr::Attribute(
Box::new(Expr::Name(acc.into())),
"append".into(),
)),
args: vec![elt.clone()],
keywords: vec![],
})
.into(),
CompKind::Set => StmtKind::Expr(Expr::Call {
func: Box::new(Expr::Attribute(
Box::new(Expr::Name(acc.into())),
"add".into(),
)),
args: vec![elt.clone()],
keywords: vec![],
})
.into(),
CompKind::Dict => StmtKind::Assign {
targets: vec![Expr::Subscript(
Box::new(Expr::Name(acc.into())),
Box::new(elt.clone()),
)],
value: val.unwrap().clone(),
}
.into(),
};
let mut inner = vec![add_stmt];
inner = wrap_comp_clauses(inner, comps);
let mut body = vec![StmtKind::Assign {
targets: vec![Expr::Name(acc.into())],
value: empty,
}
.into()];
for decl in self.comp_walrus_decls(elt, val, comps) {
body.insert(0, decl);
}
body.extend(inner);
body.push(StmtKind::Return(Some(Expr::Name(acc.into()))).into());
let is_async = comps.iter().any(|c| c.is_async);
self.emit_comp_call(b, "<comp>", body, &comps[0].iter, is_async)
}
fn comp_walrus_decls(
&self,
elt: &Expr,
val: Option<&Expr>,
comps: &[Comprehension],
) -> Vec<Stmt> {
let mut names: Vec<String> = Vec::new();
collect_walrus_targets(elt, &mut names);
if let Some(v) = val {
collect_walrus_targets(v, &mut names);
}
for c in comps {
for cond in &c.ifs {
collect_walrus_targets(cond, &mut names);
}
}
names.dedup();
if names.is_empty() {
return vec![];
}
let kind = if self.fn_depth == 0 {
StmtKind::Global(names)
} else {
StmtKind::Nonlocal(names)
};
vec![kind.into()]
}
fn compile_genexp(
&mut self,
b: &mut ChunkBuilder,
elt: &Expr,
comps: &[Comprehension],
) -> Result<(), String> {
let yield_stmt: Stmt = StmtKind::Expr(Expr::Yield(Some(Box::new(elt.clone())))).into();
let body = wrap_comp_clauses(vec![yield_stmt], comps);
self.emit_comp_call(b, "<genexpr>", body, &comps[0].iter, false)
}
fn emit_comp_call(
&mut self,
b: &mut ChunkBuilder,
name: &str,
body: Vec<Stmt>,
outer_iter: &Expr,
is_async: bool,
) -> Result<(), String> {
let params = Params {
names: vec![".0".into()],
..Params::default()
};
let def_id =
self.build_function_ex(name, ¶ms, &body, is_async, ScopeKind::Comprehension)?;
self.emit_make_func(b, def_id, ¶ms)?; self.compile_expr(b, outer_iter)?; b.emit(Op::CallBuiltin(ops::CALL_VALUE, 2), 0); if is_async {
b.emit(Op::CallBuiltin(ops::AWAIT, 1), 0);
}
Ok(())
}
fn compile_match(
&mut self,
b: &mut ChunkBuilder,
subject: &Expr,
cases: &[MatchCase],
) -> Result<(), String> {
for (i, case) in cases.iter().enumerate() {
let mut names = Vec::new();
let allow_irrefutable = case.guard.is_some() || i == cases.len() - 1;
validate_pattern(&case.pattern, &mut names, allow_irrefutable)?;
}
let subj = format!(".match{}", self.tmp);
self.tmp += 1;
self.compile_expr(b, subject)?;
self.store_name(b, &subj);
let mut end_jumps = Vec::new();
for case in cases {
let mut fails = Vec::new();
self.load_local(b, &subj);
self.compile_pattern(b, &case.pattern, &mut fails)?;
if let Some(g) = &case.guard {
self.compile_condition(b, g)?;
let jf = b.emit(Op::JumpIfFalse(0), 0);
fails.push(jf);
}
self.compile_stmts(b, &case.body)?;
end_jumps.push(b.emit(Op::Jump(0), 0));
let next = b.current_pos();
for f in fails {
b.patch_jump(f, next);
}
}
let end = b.current_pos();
for j in end_jumps {
b.patch_jump(j, end);
}
Ok(())
}
fn load_local(&self, b: &mut ChunkBuilder, name: &str) {
self.name_const(b, name);
b.emit(Op::CallBuiltin(ops::GETLOCAL, 1), 0);
}
fn compile_pattern(
&mut self,
b: &mut ChunkBuilder,
pat: &Pattern,
fails: &mut Vec<usize>,
) -> Result<(), String> {
match pat {
Pattern::Wildcard => {
b.emit(Op::Pop, 0);
}
Pattern::Capture(name) => {
self.compile_assign(b, &Expr::Name(name.clone()))?;
}
Pattern::Value(e) => {
self.compile_expr(b, e)?; if matches!(e, Expr::None | Expr::True | Expr::False) {
b.emit(Op::CallBuiltin(ops::IS, 2), 0); } else {
b.emit(Op::NumEq, 0); }
let jf = b.emit(Op::JumpIfFalse(0), 0);
fails.push(jf);
}
Pattern::As(inner, name) => {
self.compile_assign(b, &Expr::Name(name.clone()))?;
self.load_local(b, name);
self.compile_pattern(b, inner, fails)?;
}
Pattern::Or(alts) => {
let orv = format!(".or{}", self.tmp);
self.tmp += 1;
self.store_name(b, &orv);
let mut succ = Vec::new();
for alt in alts {
self.load_local(b, &orv);
let mut altfails = Vec::new();
self.compile_pattern(b, alt, &mut altfails)?;
succ.push(b.emit(Op::Jump(0), 0));
let here = b.current_pos();
for f in altfails {
b.patch_jump(f, here);
}
}
fails.push(b.emit(Op::Jump(0), 0));
let after = b.current_pos();
for j in succ {
b.patch_jump(j, after);
}
}
Pattern::Star(_) => {
b.emit(Op::Pop, 0);
}
Pattern::Sequence { elems, star } => {
self.compile_seq_pattern(b, elems, *star, fails)?;
}
Pattern::Mapping { keys, rest } => {
self.compile_map_pattern(b, keys, rest, fails)?;
}
Pattern::Class { cls, pos, kw } => {
self.compile_class_pattern(b, cls, pos, kw, fails)?;
}
}
Ok(())
}
fn compile_seq_pattern(
&mut self,
b: &mut ChunkBuilder,
elems: &[Pattern],
star: Option<usize>,
fails: &mut Vec<usize>,
) -> Result<(), String> {
b.emit(Op::LoadInt(elems.len() as i64), 0);
b.emit(Op::LoadInt(star.map(|i| i as i64).unwrap_or(-1)), 0);
b.emit(Op::CallBuiltin(ops::MATCH_SEQ, 3), 0); let jf = b.emit(Op::JumpIfFalse(0), 0);
fails.push(jf);
let seqv = format!(".seq{}", self.tmp);
self.tmp += 1;
self.store_name(b, &seqv); for (k, sub) in elems.iter().enumerate() {
self.load_local(b, &seqv);
b.emit(Op::LoadInt(k as i64), 0);
b.emit(Op::CallBuiltin(ops::GETITEM, 2), 0); match sub {
Pattern::Star(Some(name)) => {
self.compile_assign(b, &Expr::Name(name.clone()))?;
}
Pattern::Star(None) => {
b.emit(Op::Pop, 0);
}
_ => self.compile_pattern(b, sub, fails)?,
}
}
Ok(())
}
fn compile_map_pattern(
&mut self,
b: &mut ChunkBuilder,
keys: &[(Expr, Pattern)],
rest: &Option<String>,
fails: &mut Vec<usize>,
) -> Result<(), String> {
let mapv = format!(".map{}", self.tmp);
self.tmp += 1;
self.store_name(b, &mapv);
self.load_local(b, &mapv);
b.emit(Op::CallBuiltin(ops::MATCH_MAP_CHECK, 1), 0); let jf = b.emit(Op::JumpIfFalse(0), 0);
fails.push(jf);
for (keyexpr, sub) in keys {
self.load_local(b, &mapv);
self.compile_expr(b, keyexpr)?; b.emit(Op::CallBuiltin(ops::MATCH_KEY, 2), 0); let jf = b.emit(Op::JumpIfFalse(0), 0);
fails.push(jf);
self.compile_pattern(b, sub, fails)?;
}
if let Some(rname) = rest {
self.load_local(b, &mapv);
for (keyexpr, _) in keys {
self.compile_expr(b, keyexpr)?;
}
b.emit(Op::CallBuiltin(ops::MKLIST, argc(keys.len())?), 0);
b.emit(Op::CallBuiltin(ops::MATCH_MAP_REST, 2), 0); self.compile_assign(b, &Expr::Name(rname.clone()))?;
}
Ok(())
}
fn compile_class_pattern(
&mut self,
b: &mut ChunkBuilder,
cls: &Expr,
pos: &[Pattern],
kw: &[(String, Pattern)],
fails: &mut Vec<usize>,
) -> Result<(), String> {
self.compile_expr(b, cls)?; b.emit(Op::LoadInt(pos.len() as i64), 0);
for (name, _) in kw {
self.strlit(b, name);
}
b.emit(Op::CallBuiltin(ops::MATCH_CLASS, argc(3 + kw.len())?), 0); let jf = b.emit(Op::JumpIfFalse(0), 0);
fails.push(jf);
let clsv = format!(".cls{}", self.tmp);
self.tmp += 1;
self.store_name(b, &clsv);
let subs: Vec<&Pattern> = pos.iter().chain(kw.iter().map(|(_, p)| p)).collect();
for (k, sub) in subs.iter().enumerate() {
self.load_local(b, &clsv);
b.emit(Op::LoadInt(k as i64), 0);
b.emit(Op::CallBuiltin(ops::GETITEM, 2), 0); self.compile_pattern(b, sub, fails)?;
}
Ok(())
}
}
fn validate_pattern(
pat: &Pattern,
bound: &mut Vec<String>,
allow_irrefutable: bool,
) -> Result<(), String> {
match pat {
Pattern::Value(_) | Pattern::Star(None) => Ok(()),
Pattern::Wildcard => {
if !allow_irrefutable {
return Err(
"SyntaxError: wildcard makes remaining patterns unreachable".to_string()
);
}
Ok(())
}
Pattern::Capture(name) => {
if !allow_irrefutable {
return Err(format!(
"SyntaxError: name capture '{name}' makes remaining patterns unreachable"
));
}
bind_pattern_name(name, bound)
}
Pattern::Star(Some(name)) => bind_pattern_name(name, bound),
Pattern::As(inner, name) => {
validate_pattern(inner, bound, allow_irrefutable)?;
bind_pattern_name(name, bound)
}
Pattern::Sequence { elems, .. } => {
for e in elems {
validate_pattern(e, bound, true)?;
}
Ok(())
}
Pattern::Mapping { keys, rest } => {
for i in 0..keys.len() {
for j in 0..i {
if keys[i].0 == keys[j].0 {
return Err(format!(
"SyntaxError: mapping pattern checks duplicate key ({})",
pattern_key_repr(&keys[i].0)
));
}
}
}
for (_, p) in keys {
validate_pattern(p, bound, true)?;
}
if let Some(r) = rest {
bind_pattern_name(r, bound)?;
}
Ok(())
}
Pattern::Class { pos, kw, .. } => {
for i in 0..kw.len() {
for j in 0..i {
if kw[i].0 == kw[j].0 {
return Err(format!(
"SyntaxError: attribute name repeated in class pattern: {}",
kw[i].0
));
}
}
}
for p in pos {
validate_pattern(p, bound, true)?;
}
for (_, p) in kw {
validate_pattern(p, bound, true)?;
}
Ok(())
}
Pattern::Or(alts) => {
let last = alts.len().saturating_sub(1);
let base = bound.len();
let mut expected: Option<Vec<String>> = None;
for (i, alt) in alts.iter().enumerate() {
let mut alt_bound = bound.clone();
validate_pattern(alt, &mut alt_bound, allow_irrefutable && i == last)?;
let mut added: Vec<String> = alt_bound[base..].to_vec();
added.sort();
match &expected {
None => expected = Some(added),
Some(exp) if *exp != added => {
return Err("SyntaxError: alternative patterns bind different names".into())
}
_ => {}
}
}
if let Some(exp) = expected {
bound.extend(exp);
}
Ok(())
}
}
}
fn bind_pattern_name(name: &str, bound: &mut Vec<String>) -> Result<(), String> {
if bound.iter().any(|n| n == name) {
return Err(format!(
"SyntaxError: multiple assignments to name '{name}' in pattern"
));
}
bound.push(name.to_string());
Ok(())
}
fn pattern_key_repr(e: &Expr) -> String {
let e = e.unspanned();
match e {
Expr::Str(s) => format!("'{s}'"),
Expr::Int(n) => n.to_string(),
Expr::BigInt(s) => s.clone(),
Expr::Float(f) => f.to_string(),
Expr::None => "None".into(),
Expr::True => "True".into(),
Expr::False => "False".into(),
_ => "...".into(),
}
}
fn wrap_comp_clauses(mut inner: Vec<Stmt>, comps: &[Comprehension]) -> Vec<Stmt> {
for (i, comp) in comps.iter().enumerate().rev() {
for cond in comp.ifs.iter().rev() {
inner = vec![StmtKind::If {
test: cond.clone(),
body: inner,
orelse: vec![],
}
.into()];
}
let iter = if i == 0 {
Expr::Name(".0".into())
} else {
(*comp.iter).clone()
};
inner = vec![StmtKind::For {
target: (*comp.target).clone(),
iter,
body: inner,
orelse: vec![],
is_async: comp.is_async,
}
.into()];
}
inner
}
fn const_literal_type(e: &Expr) -> Option<&'static str> {
let e = e.unspanned();
match e {
Expr::Int(_) | Expr::BigInt(_) => Some("int"),
Expr::Float(_) => Some("float"),
Expr::Complex(_) => Some("complex"),
Expr::Str(_) => Some("str"),
Expr::Bytes(_) => Some("bytes"),
Expr::Tuple(items) if items.iter().all(|x| const_literal_type(x).is_some()) => {
Some("tuple")
}
_ => None,
}
}
fn literal_container_type(e: &Expr) -> Option<&'static str> {
let e = e.unspanned();
match e {
Expr::List(_) => Some("list"),
Expr::Tuple(_) => Some("tuple"),
Expr::Str(_) => Some("str"),
Expr::Bytes(_) => Some("bytes"),
_ => None,
}
}
fn compile_time_type(e: &Expr) -> Option<&'static str> {
let e = e.unspanned();
match e {
Expr::None => Some("NoneType"),
Expr::True | Expr::False => Some("bool"),
Expr::Ellipsis => Some("ellipsis"),
Expr::Int(_) | Expr::BigInt(_) => Some("int"),
Expr::Float(_) => Some("float"),
Expr::Complex(_) => Some("complex"),
Expr::Str(_) => Some("str"),
Expr::Bytes(_) => Some("bytes"),
Expr::FString(parts) => parts
.iter()
.all(|p| matches!(p, FStrPart::Lit(_)))
.then_some("str"),
Expr::List(_) => Some("list"),
Expr::Tuple(_) => Some("tuple"),
Expr::Set(_) => Some("set"),
Expr::Dict(_) => Some("dict"),
_ => None,
}
}
fn bad_index_type(e: &Expr) -> Option<&'static str> {
if matches!(e.unspanned(), Expr::Slice { .. }) {
return None;
}
compile_time_type(e).filter(|t| *t != "int" && *t != "bool")
}
fn is_ann_assign(s: &Stmt) -> bool {
match &s.kind {
StmtKind::AnnAssign { target, .. } => matches!(target.unspanned(), Expr::Name(_)),
StmtKind::If { body, orelse, .. } => {
body.iter().any(is_ann_assign) || orelse.iter().any(is_ann_assign)
}
StmtKind::For { body, orelse, .. } | StmtKind::While { body, orelse, .. } => {
body.iter().any(is_ann_assign) || orelse.iter().any(is_ann_assign)
}
StmtKind::With { body, .. } => body.iter().any(is_ann_assign),
StmtKind::Match { cases, .. } => cases.iter().any(|c| c.body.iter().any(is_ann_assign)),
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
} => {
body.iter().any(is_ann_assign)
|| orelse.iter().any(is_ann_assign)
|| finalbody.iter().any(is_ann_assign)
|| handlers.iter().any(|h| h.body.iter().any(is_ann_assign))
}
_ => false,
}
}
fn param_names(params: &Params) -> Vec<String> {
let mut v = params.names.clone();
if let Some(s) = ¶ms.star {
if !s.is_empty() {
v.push(s.clone());
}
}
v.extend(params.kwonly.iter().cloned());
if let Some(k) = ¶ms.kwargs {
v.push(k.clone());
}
v
}
fn fn_slots_allowed(body: &[Stmt]) -> bool {
if body_has_yield(body) {
return false;
}
let mut refs: HashSet<String> = HashSet::new();
for s in body {
collect_names_stmt(s, &mut refs);
}
if refs
.iter()
.any(|n| matches!(n.as_str(), "locals" | "vars" | "eval" | "exec" | "dir"))
{
return false;
}
body.iter().all(stmt_slot_safe)
}
fn stmt_slot_safe(s: &Stmt) -> bool {
let exprs_ok = |es: &[&Expr]| es.iter().all(|e| expr_slot_safe(e));
match &s.kind {
StmtKind::FuncDef { .. }
| StmtKind::ClassDef { .. }
| StmtKind::Try { .. }
| StmtKind::With { .. }
| StmtKind::Global(_)
| StmtKind::Nonlocal(_)
| StmtKind::Delete(_) => false,
StmtKind::For {
target,
iter,
body,
orelse,
is_async,
} => {
!*is_async
&& !loop_needs_signal(body)
&& exprs_ok(&[target, iter])
&& body.iter().all(stmt_slot_safe)
&& orelse.iter().all(stmt_slot_safe)
}
StmtKind::While { test, body, orelse } => {
!loop_needs_signal(body)
&& expr_slot_safe(test)
&& body.iter().all(stmt_slot_safe)
&& orelse.iter().all(stmt_slot_safe)
}
StmtKind::If { test, body, orelse } => {
expr_slot_safe(test)
&& body.iter().all(stmt_slot_safe)
&& orelse.iter().all(stmt_slot_safe)
}
StmtKind::Expr(e) | StmtKind::Return(Some(e)) => expr_slot_safe(e),
StmtKind::Assign { targets, value } => {
targets.iter().all(expr_slot_safe) && expr_slot_safe(value)
}
StmtKind::AugAssign { target, value, .. } => {
expr_slot_safe(target) && expr_slot_safe(value)
}
StmtKind::AnnAssign { target, value, .. } => {
expr_slot_safe(target) && value.as_ref().map_or(true, expr_slot_safe)
}
StmtKind::Assert { test, msg } => {
expr_slot_safe(test) && msg.as_ref().map_or(true, expr_slot_safe)
}
StmtKind::Raise { exc, cause } => {
exc.as_ref().map_or(true, expr_slot_safe) && cause.as_ref().map_or(true, expr_slot_safe)
}
StmtKind::Return(None) | StmtKind::Pass | StmtKind::Break | StmtKind::Continue => true,
_ => false,
}
}
fn expr_slot_safe(e: &Expr) -> bool {
let all = |es: &[Expr]| es.iter().all(expr_slot_safe);
match e.unspanned() {
Expr::Lambda { .. }
| Expr::ListComp(..)
| Expr::SetComp(..)
| Expr::DictComp(..)
| Expr::GenExp(..)
| Expr::Yield(_)
| Expr::YieldFrom(_)
| Expr::Await(_) => false,
Expr::List(xs) | Expr::Tuple(xs) | Expr::Set(xs) | Expr::BoolOp(_, xs) => all(xs),
Expr::Dict(pairs) => pairs
.iter()
.all(|(k, v)| k.as_ref().map_or(true, expr_slot_safe) && expr_slot_safe(v)),
Expr::Starred(x) | Expr::UnaryOp(_, x) | Expr::Attribute(x, _) => expr_slot_safe(x),
Expr::BinOp(_, a, b) | Expr::Subscript(a, b) | Expr::NamedExpr(a, b) => {
expr_slot_safe(a) && expr_slot_safe(b)
}
Expr::Compare(a, rest) => expr_slot_safe(a) && rest.iter().all(|(_, x)| expr_slot_safe(x)),
Expr::IfExp { test, body, orelse } => {
expr_slot_safe(test) && expr_slot_safe(body) && expr_slot_safe(orelse)
}
Expr::Call {
func,
args,
keywords,
} => expr_slot_safe(func) && all(args) && keywords.iter().all(|k| expr_slot_safe(&k.value)),
Expr::Slice { lo, hi, step } => {
lo.as_ref().map_or(true, |x| expr_slot_safe(x))
&& hi.as_ref().map_or(true, |x| expr_slot_safe(x))
&& step.as_ref().map_or(true, |x| expr_slot_safe(x))
}
Expr::FString(parts) | Expr::TString(parts) => parts.iter().all(|p| match p {
FStrPart::Lit(_) => true,
FStrPart::Expr { expr, spec, .. } => {
expr_slot_safe(expr)
&& spec.iter().all(|s| match s {
FStrPart::Lit(_) => true,
FStrPart::Expr { expr, .. } => expr_slot_safe(expr),
})
}
}),
_ => true,
}
}
fn provably_bound_locals(body: &[Stmt]) -> HashSet<String> {
let mut bound: HashSet<String> = HashSet::new();
let mut seen: HashSet<String> = HashSet::new();
for s in body {
if let StmtKind::Assign { targets, value } = &s.kind {
let mut vrefs = HashSet::new();
collect_names_expr(value, &mut vrefs);
for n in vrefs {
seen.insert(n);
}
for t in targets {
if let Expr::Name(n) = t.unspanned() {
if !seen.contains(n) {
bound.insert(n.clone());
}
}
}
}
let mut refs = HashSet::new();
collect_names_stmt(s, &mut refs);
for n in refs {
seen.insert(n);
}
}
bound
}
fn scope_locals(body: &[Stmt]) -> Vec<String> {
let mut bound = HashSet::new();
let mut gdecl = HashSet::new();
let mut ndecl = HashSet::new();
for s in body {
collect_bound_stmt(s, &mut bound);
collect_scope_decls_stmt(s, &mut gdecl, &mut ndecl);
}
let mut out: Vec<String> = bound
.into_iter()
.filter(|n| !gdecl.contains(n) && !ndecl.contains(n))
.collect();
out.sort();
out
}
fn scope_freevars(params: &Params, body: &[Stmt], enclosing: &[HashSet<String>]) -> Vec<String> {
let mut bound_here: HashSet<String> = scope_locals(body).into_iter().collect();
for p in param_names(params) {
bound_here.insert(p);
}
let mut gdecl = HashSet::new();
let mut ndecl = HashSet::new();
for s in body {
collect_scope_decls_stmt(s, &mut gdecl, &mut ndecl);
}
let enclosing_all: HashSet<String> = enclosing.iter().flatten().cloned().collect();
let mut refs: HashSet<String> = HashSet::new();
for s in body {
collect_names_stmt(s, &mut refs);
}
for n in &ndecl {
refs.insert(n.clone());
}
let mut fv: Vec<String> = refs
.into_iter()
.filter(|n| enclosing_all.contains(n) && !bound_here.contains(n) && !gdecl.contains(n))
.collect();
fv.sort();
fv
}
fn collect_names_expr(e: &Expr, out: &mut HashSet<String>) {
match e.unspanned() {
Expr::Name(n) => {
out.insert(n.clone());
}
Expr::List(xs) | Expr::Tuple(xs) | Expr::Set(xs) | Expr::BoolOp(_, xs) => {
for x in xs {
collect_names_expr(x, out);
}
}
Expr::Dict(pairs) => {
for (k, v) in pairs {
if let Some(k) = k {
collect_names_expr(k, out);
}
collect_names_expr(v, out);
}
}
Expr::Starred(x)
| Expr::UnaryOp(_, x)
| Expr::YieldFrom(x)
| Expr::Await(x)
| Expr::Attribute(x, _) => collect_names_expr(x, out),
Expr::Yield(Some(x)) => collect_names_expr(x, out),
Expr::Yield(None) => {}
Expr::BinOp(_, a, b) | Expr::Subscript(a, b) | Expr::NamedExpr(a, b) => {
collect_names_expr(a, out);
collect_names_expr(b, out);
}
Expr::Compare(a, rest) => {
collect_names_expr(a, out);
for (_, x) in rest {
collect_names_expr(x, out);
}
}
Expr::IfExp { test, body, orelse } => {
collect_names_expr(test, out);
collect_names_expr(body, out);
collect_names_expr(orelse, out);
}
Expr::Call {
func,
args,
keywords,
} => {
collect_names_expr(func, out);
for a in args {
collect_names_expr(a, out);
}
for kw in keywords {
collect_names_expr(&kw.value, out);
}
}
Expr::Slice { lo, hi, step } => {
for o in [lo, hi, step].into_iter().flatten() {
collect_names_expr(o, out);
}
}
Expr::Lambda { params, body } => {
for d in ¶ms.defaults {
collect_names_expr(d, out);
}
for d in params.kwonly_defaults.iter().flatten() {
collect_names_expr(d, out);
}
collect_names_expr(body, out);
}
Expr::ListComp(elt, comps) | Expr::SetComp(elt, comps) | Expr::GenExp(elt, comps) => {
collect_names_expr(elt, out);
collect_names_comps(comps, out);
}
Expr::DictComp(k, v, comps) => {
collect_names_expr(k, out);
collect_names_expr(v, out);
collect_names_comps(comps, out);
}
Expr::FString(parts) | Expr::TString(parts) => collect_names_fstr(parts, out),
_ => {} }
}
fn collect_names_comps(comps: &[Comprehension], out: &mut HashSet<String>) {
for c in comps {
collect_names_expr(&c.target, out);
collect_names_expr(&c.iter, out);
for cond in &c.ifs {
collect_names_expr(cond, out);
}
}
}
fn collect_names_fstr(parts: &[FStrPart], out: &mut HashSet<String>) {
for p in parts {
if let FStrPart::Expr { expr, spec, .. } = p {
collect_names_expr(expr, out);
collect_names_fstr(spec, out);
}
}
}
fn check_except_star(handlers: &[ExceptHandler]) -> Result<(), String> {
if !handlers.iter().any(|h| h.star) {
return Ok(());
}
if handlers.iter().any(|h| !h.star) {
return Err(
"SyntaxError: cannot have both 'except' and 'except*' on the same 'try'".to_string(),
);
}
for h in handlers {
if h.typ.is_none() {
return Err("SyntaxError: expected one or more exception types".to_string());
}
if handler_escapes(&h.body) {
return Err(
"SyntaxError: 'break', 'continue' and 'return' cannot appear in an except* block"
.to_string(),
);
}
}
Ok(())
}
fn handler_escapes(body: &[Stmt]) -> bool {
body.iter().any(|s| match &s.kind {
StmtKind::Break | StmtKind::Continue | StmtKind::Return(_) => true,
StmtKind::If { body, orelse, .. } => handler_escapes(body) || handler_escapes(orelse),
StmtKind::With { body, .. } => handler_escapes(body),
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
} => {
handler_escapes(body)
|| handlers.iter().any(|h| handler_escapes(&h.body))
|| handler_escapes(orelse)
|| handler_escapes(finalbody)
}
StmtKind::Match { cases, .. } => cases.iter().any(|c| handler_escapes(&c.body)),
StmtKind::While { body, orelse, .. } | StmtKind::For { body, orelse, .. } => {
has_return(body) || has_return(orelse)
}
_ => false,
})
}
fn has_return(body: &[Stmt]) -> bool {
body.iter().any(|s| match &s.kind {
StmtKind::Return(_) => true,
StmtKind::If { body, orelse, .. } => has_return(body) || has_return(orelse),
StmtKind::While { body, orelse, .. } | StmtKind::For { body, orelse, .. } => {
has_return(body) || has_return(orelse)
}
StmtKind::With { body, .. } => has_return(body),
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
} => {
has_return(body)
|| handlers.iter().any(|h| has_return(&h.body))
|| has_return(orelse)
|| has_return(finalbody)
}
StmtKind::Match { cases, .. } => cases.iter().any(|c| has_return(&c.body)),
_ => false,
})
}
fn expr_children(e: &Expr) -> Vec<&Expr> {
let mut out: Vec<&Expr> = Vec::new();
match e.unspanned() {
Expr::List(xs) | Expr::Tuple(xs) | Expr::Set(xs) | Expr::BoolOp(_, xs) => {
out.extend(xs.iter())
}
Expr::Dict(pairs) => {
for (k, v) in pairs {
out.extend(k.iter());
out.push(v);
}
}
Expr::Starred(x)
| Expr::UnaryOp(_, x)
| Expr::YieldFrom(x)
| Expr::Await(x)
| Expr::Attribute(x, _)
| Expr::Yield(Some(x)) => out.push(x),
Expr::BinOp(_, a, b) | Expr::Subscript(a, b) | Expr::NamedExpr(a, b) => {
out.push(a);
out.push(b);
}
Expr::Compare(a, rest) => {
out.push(a);
out.extend(rest.iter().map(|(_, x)| x));
}
Expr::IfExp { test, body, orelse } => out.extend([&**test, &**body, &**orelse]),
Expr::Call {
func,
args,
keywords,
} => {
out.push(func);
out.extend(args.iter());
out.extend(keywords.iter().map(|k| &k.value));
}
Expr::Slice { lo, hi, step } => {
out.extend([lo, hi, step].into_iter().flatten().map(|b| &**b))
}
Expr::Lambda { params, body } => {
out.extend(params.defaults.iter());
out.extend(params.kwonly_defaults.iter().flatten());
out.push(body);
}
Expr::ListComp(elt, comps) | Expr::SetComp(elt, comps) | Expr::GenExp(elt, comps) => {
out.push(elt);
push_comp_children(comps, &mut out);
}
Expr::DictComp(k, v, comps) => {
out.push(k);
out.push(v);
push_comp_children(comps, &mut out);
}
Expr::FString(parts) | Expr::TString(parts) => push_fstr_children(parts, &mut out),
_ => {} }
out
}
fn push_comp_children<'a>(comps: &'a [Comprehension], out: &mut Vec<&'a Expr>) {
for c in comps {
out.push(&c.target);
out.push(&c.iter);
out.extend(c.ifs.iter());
}
}
fn push_fstr_children<'a>(parts: &'a [FStrPart], out: &mut Vec<&'a Expr>) {
for p in parts {
if let FStrPart::Expr { expr, spec, .. } = p {
out.push(expr);
push_fstr_children(spec, out);
}
}
}
fn expr_has_walrus(e: &Expr) -> bool {
matches!(e.unspanned(), Expr::NamedExpr(..))
|| expr_children(e).into_iter().any(expr_has_walrus)
}
fn collect_target_names(t: &Expr, out: &mut Vec<String>) {
match t.unspanned() {
Expr::Name(n) => out.push(n.clone()),
Expr::Tuple(xs) | Expr::List(xs) => {
for x in xs {
collect_target_names(x, out);
}
}
Expr::Starred(x) => collect_target_names(x, out),
_ => {}
}
}
fn check_comprehension_walrus(
elts: &[&Expr],
comps: &[Comprehension],
outer: &[String],
) -> Result<(), String> {
for c in comps {
if expr_has_walrus(&c.iter) {
return Err("SyntaxError: assignment expression cannot be used in a \
comprehension iterable expression"
.to_string());
}
}
let mut bound = outer.to_vec();
for c in comps {
collect_target_names(&c.target, &mut bound);
}
for e in elts
.iter()
.copied()
.chain(comps.iter().flat_map(|c| c.ifs.iter()))
{
check_walrus_rebind(e, &bound)?;
}
Ok(())
}
fn check_walrus_rebind(e: &Expr, bound: &[String]) -> Result<(), String> {
match e.unspanned() {
Expr::Lambda { params, .. } => {
for d in params
.defaults
.iter()
.chain(params.kwonly_defaults.iter().flatten())
{
check_walrus_rebind(d, bound)?;
}
return Ok(());
}
Expr::ListComp(elt, comps) | Expr::SetComp(elt, comps) | Expr::GenExp(elt, comps) => {
return check_comprehension_walrus(&[elt], comps, bound)
}
Expr::DictComp(k, v, comps) => return check_comprehension_walrus(&[k, v], comps, bound),
Expr::NamedExpr(target, _) => {
if let Expr::Name(n) = target.unspanned() {
if bound.iter().any(|b| b == n) {
return Err(format!(
"SyntaxError: assignment expression cannot rebind \
comprehension iteration variable '{n}'"
));
}
}
}
_ => {}
}
for c in expr_children(e) {
check_walrus_rebind(c, bound)?;
}
Ok(())
}
fn collect_names_stmt(s: &Stmt, out: &mut HashSet<String>) {
match &s.kind {
StmtKind::Expr(e) => collect_names_expr(e, out),
StmtKind::Assign { targets, value } => {
for t in targets {
collect_names_expr(t, out);
}
collect_names_expr(value, out);
}
StmtKind::AugAssign { target, value, .. } => {
collect_names_expr(target, out);
collect_names_expr(value, out);
}
StmtKind::AnnAssign { target, value, .. } => {
collect_names_expr(target, out);
if let Some(v) = value {
collect_names_expr(v, out);
}
}
StmtKind::If { test, body, orelse } | StmtKind::While { test, body, orelse } => {
collect_names_expr(test, out);
for st in body.iter().chain(orelse) {
collect_names_stmt(st, out);
}
}
StmtKind::For {
target,
iter,
body,
orelse,
..
} => {
collect_names_expr(target, out);
collect_names_expr(iter, out);
for st in body.iter().chain(orelse) {
collect_names_stmt(st, out);
}
}
StmtKind::With { items, body, .. } => {
for it in items {
collect_names_expr(&it.context, out);
if let Some(a) = &it.vars {
collect_names_expr(a, out);
}
}
for st in body {
collect_names_stmt(st, out);
}
}
StmtKind::FuncDef {
params,
body,
decorators,
..
} => {
for d in ¶ms.defaults {
collect_names_expr(d, out);
}
for d in params.kwonly_defaults.iter().flatten() {
collect_names_expr(d, out);
}
for dec in decorators {
collect_names_expr(dec, out);
}
for st in body {
collect_names_stmt(st, out);
}
}
StmtKind::ClassDef {
bases,
keywords,
body,
decorators,
..
} => {
for b in bases {
collect_names_expr(b, out);
}
for kw in keywords {
collect_names_expr(&kw.value, out);
}
for dec in decorators {
collect_names_expr(dec, out);
}
for st in body {
collect_names_stmt(st, out);
}
}
StmtKind::Return(Some(e)) => collect_names_expr(e, out),
StmtKind::Delete(es) => {
for e in es {
collect_names_expr(e, out);
}
}
StmtKind::Raise { exc, cause } => {
if let Some(e) = exc {
collect_names_expr(e, out);
}
if let Some(c) = cause {
collect_names_expr(c, out);
}
}
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
} => {
for st in body {
collect_names_stmt(st, out);
}
for h in handlers {
if let Some(t) = &h.typ {
collect_names_expr(t, out);
}
for st in &h.body {
collect_names_stmt(st, out);
}
}
for st in orelse.iter().chain(finalbody) {
collect_names_stmt(st, out);
}
}
StmtKind::Assert { test, msg } => {
collect_names_expr(test, out);
if let Some(m) = msg {
collect_names_expr(m, out);
}
}
StmtKind::Match { subject, cases } => {
collect_names_expr(subject, out);
for case in cases {
if let Some(g) = &case.guard {
collect_names_expr(g, out);
}
for st in &case.body {
collect_names_stmt(st, out);
}
}
}
_ => {} }
}
fn bind_target(e: &Expr, out: &mut HashSet<String>) {
let e = e.unspanned();
match e {
Expr::Name(n) => {
out.insert(n.clone());
}
Expr::Tuple(items) | Expr::List(items) => {
for it in items {
bind_target(it, out);
}
}
Expr::Starred(x) => bind_target(x, out),
_ => {}
}
}
fn collect_bound_stmt(s: &Stmt, out: &mut HashSet<String>) {
match &s.kind {
StmtKind::Assign { targets, value } => {
for t in targets {
bind_target(t, out);
}
collect_leaked_walrus(value, out);
}
StmtKind::AugAssign { target, value, .. } => {
bind_target(target, out);
collect_leaked_walrus(value, out);
}
StmtKind::AnnAssign { target, value, .. } => {
bind_target(target, out);
if let Some(v) = value {
collect_leaked_walrus(v, out);
}
}
StmtKind::For {
target,
iter,
body,
orelse,
..
} => {
bind_target(target, out);
collect_leaked_walrus(iter, out);
for s in body.iter().chain(orelse) {
collect_bound_stmt(s, out);
}
}
StmtKind::While { test, body, orelse } => {
collect_leaked_walrus(test, out);
for s in body.iter().chain(orelse) {
collect_bound_stmt(s, out);
}
}
StmtKind::If { test, body, orelse } => {
collect_leaked_walrus(test, out);
for s in body.iter().chain(orelse) {
collect_bound_stmt(s, out);
}
}
StmtKind::With { items, body, .. } => {
for it in items {
collect_leaked_walrus(&it.context, out);
if let Some(v) = &it.vars {
bind_target(v, out);
}
}
for s in body {
collect_bound_stmt(s, out);
}
}
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
} => {
for s in body.iter().chain(orelse).chain(finalbody) {
collect_bound_stmt(s, out);
}
for h in handlers {
if let Some(n) = &h.name {
out.insert(n.clone());
}
for s in &h.body {
collect_bound_stmt(s, out);
}
}
}
StmtKind::FuncDef { name, .. } | StmtKind::ClassDef { name, .. } => {
out.insert(name.clone());
}
StmtKind::Import(aliases) => {
for a in aliases {
let n = a
.asname
.clone()
.unwrap_or_else(|| a.name.split('.').next().unwrap_or(&a.name).to_string());
out.insert(n);
}
}
StmtKind::ImportFrom { names, .. } => {
for a in names {
out.insert(a.asname.clone().unwrap_or_else(|| a.name.clone()));
}
}
StmtKind::Match { subject, cases } => {
collect_leaked_walrus(subject, out);
for c in cases {
let mut names = Vec::new();
let _ = validate_pattern(&c.pattern, &mut names, true);
for n in names {
out.insert(n);
}
if let Some(g) = &c.guard {
collect_leaked_walrus(g, out);
}
for s in &c.body {
collect_bound_stmt(s, out);
}
}
}
StmtKind::Expr(e) | StmtKind::Return(Some(e)) => collect_leaked_walrus(e, out),
StmtKind::Delete(targets) => {
for t in targets {
bind_target(t, out);
}
}
_ => {}
}
}
fn collect_scope_decls_stmt(s: &Stmt, g: &mut HashSet<String>, nl: &mut HashSet<String>) {
match &s.kind {
StmtKind::Global(names) => {
for n in names {
g.insert(n.clone());
}
}
StmtKind::Nonlocal(names) => {
for n in names {
nl.insert(n.clone());
}
}
StmtKind::If { body, orelse, .. }
| StmtKind::While { body, orelse, .. }
| StmtKind::For { body, orelse, .. } => {
for s in body.iter().chain(orelse) {
collect_scope_decls_stmt(s, g, nl);
}
}
StmtKind::With { body, .. } => {
for s in body {
collect_scope_decls_stmt(s, g, nl);
}
}
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
} => {
for s in body.iter().chain(orelse).chain(finalbody) {
collect_scope_decls_stmt(s, g, nl);
}
for h in handlers {
for s in &h.body {
collect_scope_decls_stmt(s, g, nl);
}
}
}
StmtKind::Match { cases, .. } => {
for c in cases {
for s in &c.body {
collect_scope_decls_stmt(s, g, nl);
}
}
}
_ => {}
}
}
fn collect_leaked_walrus(e: &Expr, out: &mut HashSet<String>) {
let e = e.unspanned();
match e {
Expr::NamedExpr(target, value) => {
if let Expr::Name(n) = target.unspanned() {
out.insert(n.clone());
}
collect_leaked_walrus(value, out);
}
Expr::BoolOp(_, items) | Expr::List(items) | Expr::Tuple(items) | Expr::Set(items) => {
for it in items {
collect_leaked_walrus(it, out);
}
}
Expr::Dict(pairs) => {
for (k, v) in pairs {
if let Some(k) = k {
collect_leaked_walrus(k, out);
}
collect_leaked_walrus(v, out);
}
}
Expr::UnaryOp(_, x)
| Expr::Starred(x)
| Expr::Await(x)
| Expr::YieldFrom(x)
| Expr::Yield(Some(x)) => collect_leaked_walrus(x, out),
Expr::BinOp(_, a, b) => {
collect_leaked_walrus(a, out);
collect_leaked_walrus(b, out);
}
Expr::Compare(a, links) => {
collect_leaked_walrus(a, out);
for (_, rhs) in links {
collect_leaked_walrus(rhs, out);
}
}
Expr::IfExp { test, body, orelse } => {
collect_leaked_walrus(test, out);
collect_leaked_walrus(body, out);
collect_leaked_walrus(orelse, out);
}
Expr::Call {
func,
args,
keywords,
} => {
collect_leaked_walrus(func, out);
for a in args {
collect_leaked_walrus(a, out);
}
for kw in keywords {
collect_leaked_walrus(&kw.value, out);
}
}
Expr::Attribute(x, _) => collect_leaked_walrus(x, out),
Expr::Subscript(a, b) => {
collect_leaked_walrus(a, out);
collect_leaked_walrus(b, out);
}
Expr::Slice { lo, hi, step } => {
for p in [lo, hi, step].into_iter().flatten() {
collect_leaked_walrus(p, out);
}
}
Expr::ListComp(elt, comps) | Expr::SetComp(elt, comps) | Expr::GenExp(elt, comps) => {
collect_leaked_walrus(elt, out);
for c in comps {
collect_leaked_walrus(&c.iter, out);
for cond in &c.ifs {
collect_leaked_walrus(cond, out);
}
}
}
Expr::DictComp(k, v, comps) => {
collect_leaked_walrus(k, out);
collect_leaked_walrus(v, out);
for c in comps {
collect_leaked_walrus(&c.iter, out);
for cond in &c.ifs {
collect_leaked_walrus(cond, out);
}
}
}
Expr::Lambda { .. } => {}
_ => {}
}
}
fn collect_walrus_targets(e: &Expr, out: &mut Vec<String>) {
let e = e.unspanned();
match e {
Expr::NamedExpr(target, value) => {
if let Expr::Name(n) = target.unspanned() {
if !out.contains(n) {
out.push(n.clone());
}
}
collect_walrus_targets(value, out);
}
Expr::BoolOp(_, items) | Expr::List(items) | Expr::Tuple(items) | Expr::Set(items) => {
for it in items {
collect_walrus_targets(it, out);
}
}
Expr::Dict(pairs) => {
for (k, v) in pairs {
if let Some(k) = k {
collect_walrus_targets(k, out);
}
collect_walrus_targets(v, out);
}
}
Expr::UnaryOp(_, x) | Expr::Starred(x) | Expr::Await(x) | Expr::YieldFrom(x) => {
collect_walrus_targets(x, out)
}
Expr::Yield(Some(x)) => collect_walrus_targets(x, out),
Expr::BinOp(_, a, b) => {
collect_walrus_targets(a, out);
collect_walrus_targets(b, out);
}
Expr::Compare(a, links) => {
collect_walrus_targets(a, out);
for (_, rhs) in links {
collect_walrus_targets(rhs, out);
}
}
Expr::IfExp { test, body, orelse } => {
collect_walrus_targets(test, out);
collect_walrus_targets(body, out);
collect_walrus_targets(orelse, out);
}
Expr::Call {
func,
args,
keywords,
} => {
collect_walrus_targets(func, out);
for a in args {
collect_walrus_targets(a, out);
}
for kw in keywords {
collect_walrus_targets(&kw.value, out);
}
}
Expr::Attribute(x, _) => collect_walrus_targets(x, out),
Expr::Subscript(a, b) => {
collect_walrus_targets(a, out);
collect_walrus_targets(b, out);
}
Expr::Slice { lo, hi, step } => {
for p in [lo, hi, step].into_iter().flatten() {
collect_walrus_targets(p, out);
}
}
Expr::Lambda { .. }
| Expr::ListComp(..)
| Expr::SetComp(..)
| Expr::DictComp(..)
| Expr::GenExp(..) => {}
_ => {}
}
}
#[derive(Clone, Copy)]
enum CompKind {
List,
Set,
Dict,
}
fn body_has_yield(body: &[Stmt]) -> bool {
body.iter().any(stmt_has_yield)
}
fn docstring(body: &[Stmt]) -> Option<String> {
match body.first().map(|s| &s.kind) {
Some(StmtKind::Expr(e)) => match e.unspanned() {
Expr::Str(s) => Some(s.clone()),
_ => None,
},
_ => None,
}
}
fn stmt_has_yield(s: &Stmt) -> bool {
match &s.kind {
StmtKind::Expr(e) | StmtKind::Return(Some(e)) => expr_has_yield(e),
StmtKind::Assign { value, .. } => expr_has_yield(value),
StmtKind::AugAssign { value, .. } => expr_has_yield(value),
StmtKind::If { body, orelse, .. } => body_has_yield(body) || body_has_yield(orelse),
StmtKind::While { body, orelse, .. } => body_has_yield(body) || body_has_yield(orelse),
StmtKind::For { body, orelse, .. } => body_has_yield(body) || body_has_yield(orelse),
StmtKind::With { body, .. } => body_has_yield(body),
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
} => {
body_has_yield(body)
|| body_has_yield(orelse)
|| body_has_yield(finalbody)
|| handlers.iter().any(|h| body_has_yield(&h.body))
}
StmtKind::Match { cases, .. } => cases.iter().any(|c| body_has_yield(&c.body)),
_ => false,
}
}
fn expr_has_yield(e: &Expr) -> bool {
matches!(e.unspanned(), Expr::Yield(_) | Expr::YieldFrom(_))
}
fn collect_finally_escapes(stmts: &[Stmt], out: &mut Vec<(u32, String)>) {
for s in stmts {
match &s.kind {
StmtKind::Return(_) => out.push((s.line, "'return' in a 'finally' block".to_string())),
StmtKind::Break => out.push((s.line, "'break' in a 'finally' block".to_string())),
StmtKind::Continue => out.push((s.line, "'continue' in a 'finally' block".to_string())),
StmtKind::If { body, orelse, .. } => {
collect_finally_escapes(body, out);
collect_finally_escapes(orelse, out);
}
StmtKind::With { body, .. } => collect_finally_escapes(body, out),
StmtKind::Match { cases, .. } => {
for c in cases {
collect_finally_escapes(&c.body, out);
}
}
StmtKind::Try {
body,
handlers,
orelse,
..
} => {
collect_finally_escapes(body, out);
collect_finally_escapes(orelse, out);
for h in handlers {
collect_finally_escapes(&h.body, out);
}
}
_ => {}
}
}
}
fn native_arith_op(op: BinOp) -> Option<Op> {
match op {
BinOp::Add => Some(Op::Add),
BinOp::Sub => Some(Op::Sub),
BinOp::Mul => Some(Op::Mul),
_ => None,
}
}
#[derive(Clone, Copy)]
enum IntNames<'a> {
All,
Only(&'a [String]),
}
impl IntNames<'_> {
fn contains(&self, name: &str) -> bool {
match self {
IntNames::All => true,
IntNames::Only(names) => names.iter().any(|n| n == name),
}
}
}
fn native_safe_value(e: &Expr, reads: &mut Vec<String>, int_names: IntNames) -> bool {
match e.unspanned() {
Expr::Int(_) => true,
Expr::Name(n) => {
if !reads.iter().any(|r| r == n) {
reads.push(n.clone());
}
true
}
Expr::UnaryOp(UnOp::Neg | UnOp::Pos, inner) => native_safe_value(inner, reads, int_names),
Expr::BinOp(BinOp::Mod, l, r) => {
native_mod_divisor(r).is_some()
&& (fused_mod_dividend(l) || provably_int(l, int_names))
&& native_safe_value(l, reads, int_names)
}
Expr::BinOp(op, l, r) => {
native_arith_op(*op).is_some()
&& native_safe_value(l, reads, int_names)
&& native_safe_value(r, reads, int_names)
}
_ => false,
}
}
fn provably_int(e: &Expr, int_names: IntNames) -> bool {
match e.unspanned() {
Expr::Int(_) => true,
Expr::Name(n) => int_names.contains(n),
Expr::UnaryOp(UnOp::Neg | UnOp::Pos, inner) => provably_int(inner, int_names),
Expr::BinOp(BinOp::Mod, l, r) => {
native_mod_divisor(r).is_some() && provably_int(l, int_names)
}
Expr::BinOp(op, l, r) => {
native_arith_op(*op).is_some()
&& provably_int(l, int_names)
&& provably_int(r, int_names)
}
_ => false,
}
}
struct NativeForPlan {
guard_jumps: Vec<usize>,
skip_generic: Option<usize>,
}
fn mul_operands(e: &Expr) -> Option<(&Expr, &Expr)> {
match e.unspanned() {
Expr::BinOp(BinOp::Mul, l, r) => Some((l, r)),
_ => None,
}
}
fn fused_mod_dividend(e: &Expr) -> bool {
match e.unspanned() {
Expr::BinOp(BinOp::Mul, _, _) => true,
Expr::BinOp(BinOp::Add, l, r) => mul_operands(l).is_some() || mul_operands(r).is_some(),
_ => false,
}
}
fn native_mod_divisor(e: &Expr) -> Option<i64> {
match literal_int(e) {
Some(k) if k.abs() >= 2 => Some(k),
_ => None,
}
}
fn literal_int(e: &Expr) -> Option<i64> {
match e.unspanned() {
Expr::Int(n) => Some(*n),
Expr::UnaryOp(UnOp::Neg, inner) => match inner.unspanned() {
Expr::Int(n) => Some(-*n),
_ => None,
},
_ => None,
}
}
fn native_range_call(iter: &Expr) -> Option<(Option<&Expr>, &Expr, i64)> {
let (func, args) = match iter.unspanned() {
Expr::Call {
func,
args,
keywords,
} if keywords.is_empty() => (func, args),
_ => return None,
};
match func.unspanned() {
Expr::Name(n) if n == "range" => {}
_ => return None,
}
match args.len() {
1 => Some((None, &args[0], 1)),
2 => Some((Some(&args[0]), &args[1], 1)),
3 => match literal_int(&args[2]) {
Some(step) if step != 0 => Some((Some(&args[0]), &args[1], step)),
_ => None,
},
_ => None,
}
}
fn push_unique(v: &mut Vec<String>, name: &str) {
if !v.iter().any(|x| x == name) {
v.push(name.to_string());
}
}
fn native_cmp_op(op: CmpOp) -> Option<Op> {
match op {
CmpOp::Eq => Some(Op::NumEq),
CmpOp::Ne => Some(Op::NumNe),
CmpOp::Lt => Some(Op::NumLt),
CmpOp::Le => Some(Op::NumLe),
CmpOp::Gt => Some(Op::NumGt),
CmpOp::Ge => Some(Op::NumGe),
_ => None,
}
}
fn native_safe_cond(test: &Expr, reads: &mut Vec<String>, int_names: IntNames) -> bool {
match test.unspanned() {
Expr::Compare(lhs, rest) if rest.len() == 1 && native_cmp_op(rest[0].0).is_some() => {
native_safe_value(lhs, reads, int_names)
&& native_safe_value(&rest[0].1, reads, int_names)
}
other => native_safe_value(other, reads, int_names),
}
}
fn analyze_native_tree(
body: &[Stmt],
loop_vars: &mut Vec<String>,
reads: &mut Vec<String>,
writes: &mut Vec<String>,
int_names: IntNames,
) -> bool {
analyze_native_tree_at(body, false, loop_vars, reads, writes, int_names)
}
fn analyze_native_tree_at(
body: &[Stmt],
conditional: bool,
loop_vars: &mut Vec<String>,
reads: &mut Vec<String>,
writes: &mut Vec<String>,
int_names: IntNames,
) -> bool {
for s in body {
match &s.kind {
StmtKind::Pass => {}
StmtKind::Assign { targets, value } => {
if targets.len() != 1 {
return false;
}
let name = match targets[0].unspanned() {
Expr::Name(n) => n,
_ => return false,
};
if loop_vars.iter().any(|v| v == name) {
return false; }
if !native_safe_value(value, reads, int_names) {
return false;
}
push_unique(writes, name);
}
StmtKind::AugAssign { target, op, value } => {
let name = match target.unspanned() {
Expr::Name(n) => n,
_ => return false,
};
if loop_vars.iter().any(|v| v == name) || native_arith_op(*op).is_none() {
return false;
}
push_unique(reads, name); if !native_safe_value(value, reads, int_names) {
return false;
}
push_unique(writes, name);
}
StmtKind::If {
test,
body: tb,
orelse,
} => {
if !native_safe_cond(test, reads, int_names) {
return false;
}
if !analyze_native_tree_at(tb, true, loop_vars, reads, writes, int_names)
|| !analyze_native_tree_at(orelse, true, loop_vars, reads, writes, int_names)
{
return false;
}
}
StmtKind::While {
test,
body: wb,
orelse,
} => {
if !orelse.is_empty() || loop_needs_signal(wb) {
return false;
}
if !native_safe_cond(test, reads, int_names) {
return false;
}
if !analyze_native_tree_at(wb, true, loop_vars, reads, writes, int_names) {
return false;
}
}
StmtKind::For {
target,
iter,
body: inner,
orelse,
is_async,
} => {
if conditional || *is_async || !orelse.is_empty() || loop_needs_signal(inner) {
return false; }
let tname = match target.unspanned() {
Expr::Name(n) => n.clone(),
_ => return false,
};
let (start, stop, _step) = match native_range_call(iter) {
Some(b) => b,
None => return false,
};
if let Some(e) = start {
if !native_safe_value(e, reads, int_names) {
return false;
}
}
if !native_safe_value(stop, reads, int_names) {
return false;
}
push_unique(loop_vars, &tname);
push_unique(writes, &tname);
if !analyze_native_tree_at(inner, false, loop_vars, reads, writes, int_names) {
return false;
}
}
_ => return false,
}
}
true
}
fn reads_needing_load(e: &Expr, defined: &[String], loads: &mut Vec<String>) {
match e.unspanned() {
Expr::Name(n) => {
if !defined.iter().any(|d| d == n) {
push_unique(loads, n);
}
}
Expr::UnaryOp(_, inner) => reads_needing_load(inner, defined, loads),
Expr::BinOp(_, l, r) => {
reads_needing_load(l, defined, loads);
reads_needing_load(r, defined, loads);
}
Expr::Compare(lhs, rest) => {
reads_needing_load(lhs, defined, loads);
for (_, r) in rest {
reads_needing_load(r, defined, loads);
}
}
_ => {} }
}
fn collect_ns_loads(body: &[Stmt], defined: &mut Vec<String>, loads: &mut Vec<String>) {
for s in body {
match &s.kind {
StmtKind::Pass => {}
StmtKind::Assign { targets, value } => {
reads_needing_load(value, defined, loads);
if let Expr::Name(n) = targets[0].unspanned() {
push_unique(defined, n);
}
}
StmtKind::AugAssign { target, value, .. } => {
if let Expr::Name(n) = target.unspanned() {
if !defined.iter().any(|d| d == n) {
push_unique(loads, n); }
reads_needing_load(value, defined, loads);
push_unique(defined, n);
}
}
StmtKind::If {
test,
body: tb,
orelse,
} => {
reads_needing_load(test, defined, loads);
let mut d1 = defined.clone();
collect_ns_loads(tb, &mut d1, loads);
let mut d2 = defined.clone();
collect_ns_loads(orelse, &mut d2, loads);
}
StmtKind::While { test, body: wb, .. } => {
reads_needing_load(test, defined, loads);
let mut d = defined.clone();
collect_ns_loads(wb, &mut d, loads);
}
StmtKind::For {
target,
iter,
body: inner,
..
} => {
if let Some((start, stop, _step)) = native_range_call(iter) {
if let Some(e) = start {
reads_needing_load(e, defined, loads);
}
reads_needing_load(stop, defined, loads);
}
let mut d = defined.clone();
if let Expr::Name(n) = target.unspanned() {
push_unique(&mut d, n);
}
collect_ns_loads(inner, &mut d, loads);
}
_ => {}
}
}
}
fn native_writebacks_valid(
writes: &[String],
loads: &[String],
defined: &[String],
loop_vars: &[String],
guard_gated: bool,
) -> bool {
writes.iter().all(|w| {
loads.iter().any(|x| x == w)
|| loop_vars.iter().any(|v| v == w)
|| (guard_gated && defined.iter().any(|d| d == w))
})
}
fn loop_needs_signal(body: &[Stmt]) -> bool {
fn scan(stmts: &[Stmt], in_boundary: bool) -> bool {
stmts.iter().any(|s| match &s.kind {
StmtKind::Break | StmtKind::Continue => in_boundary,
StmtKind::If { body, orelse, .. } => {
scan(body, in_boundary) || scan(orelse, in_boundary)
}
StmtKind::Match { cases, .. } => cases.iter().any(|c| scan(&c.body, in_boundary)),
StmtKind::With { body, .. } => scan(body, true),
StmtKind::For { orelse, .. } | StmtKind::While { orelse, .. } => {
scan(orelse, in_boundary)
}
StmtKind::Try {
body,
handlers,
orelse,
finalbody,
} => {
scan(body, true)
|| scan(orelse, true)
|| scan(finalbody, true)
|| handlers.iter().any(|h| scan(&h.body, true))
}
_ => false,
})
}
scan(body, false)
}