import argparse
import os
import sys
import tempfile
def build_python(cpython_path, version):
print("Compiling python %s from repo at %s" % (version, cpython_path))
install_path = os.path.abspath(os.path.join(cpython_path, version))
ret = os.system(
f"""
cd {cpython_path}
git checkout {version}
# build in a subdirectory
mkdir -p build_{version}
cd build_{version}
../configure prefix={install_path}
make
make install
"""
)
if ret:
return ret
pip = os.path.join(install_path, "bin", "pip3" if version.startswith("v3") else "pip")
return os.system(f"{pip} install setuptools_rust wheel")
def calculate_pyruntime_offsets(cpython_path, version, configure=False):
ret = os.system(f"""cd {cpython_path} && git checkout {version}""")
if ret:
return ret
if configure:
os.system(f"cd {cpython_path} && ./configure prefix=" + os.path.abspath(os.path.join(cpython_path, version)))
program = r"""
#include "Include/Python.h"
#include "Include/internal/pystate.h"
int main(int argc, const char * argv[]) {
size_t interp_head = offsetof(_PyRuntimeState, interpreters.head);
printf("pub static INTERP_HEAD_OFFSET: usize = %i;\n", interp_head);
// tstate_current has been replaced by a thread-local variable in python 3.12
// size_t tstate_current = offsetof(_PyRuntimeState, gilstate.tstate_current);
// printf("pub static TSTATE_CURRENT: usize = %i;\n", tstate_current);
}
"""
if not os.path.isfile(os.path.join(cpython_path, "Include", "internal", "pystate.h")):
if os.path.isfile(os.path.join(cpython_path, "Include", "internal", "pycore_pystate.h")):
program = program.replace("pystate.h", "pycore_pystate.h")
else:
print("failed to find Include/internal/pystate.h in cpython directory =(")
return
with tempfile.TemporaryDirectory() as path:
if sys.platform.startswith("win"):
source_filename = os.path.join(path, "pyruntime_offsets.cpp")
exe = os.path.join("pyruntime_offsets.exe")
else:
source_filename = os.path.join(path, "pyruntime_offsets.c")
exe = os.path.join(path, "pyruntime_offsets")
with open(source_filename, "w") as o:
o.write(program)
if sys.platform.startswith("win"):
ret = os.system(f"cl {source_filename} /I {cpython_path} /I {cpython_path}\PC /I {cpython_path}\Include")
elif sys.platform.startswith("freebsd"):
ret = os.system(f"""cc {source_filename} -I {cpython_path} -I {cpython_path}/Include -o {exe}""")
else:
ret = os.system(f"""gcc {source_filename} -I {cpython_path} -I {cpython_path}/Include -o {exe}""")
if ret:
print("Failed to compile")
return ret
ret = os.system(exe)
if ret:
print("Failed to run pyruntime file")
return ret
def extract_bindings(cpython_path, version, configure=False):
print("Generating bindings for python %s from repo at %s" % (version, cpython_path))
ret = os.system(
f"""
cd {cpython_path}
git checkout {version}
# need to run configure on the current branch to generate pyconfig.h sometimes
{("./configure prefix=" + os.path.abspath(os.path.join(cpython_path, version))) if configure else ""}
echo "// autogenerated by generate_bindings.py " > bindgen_input.h
echo '#define Py_BUILD_CORE 1\n' >> bindgen_input.h
cat Include/Python.h >> bindgen_input.h
echo '#undef HAVE_STD_ATOMIC' >> bindgen_input.h
cat Include/frameobject.h >> bindgen_input.h
cat Include/internal/pycore_interp.h >> bindgen_input.h
cat Include/internal/pycore_dict.h >> bindgen_input.h
bindgen bindgen_input.h -o bindgen_output.rs \
--with-derive-default \
--no-layout-tests --no-doc-comments \
--whitelist-type PyInterpreterState \
--whitelist-type PyFrameObject \
--whitelist-type PyThreadState \
--whitelist-type PyCodeObject \
--whitelist-type PyVarObject \
--whitelist-type PyBytesObject \
--whitelist-type PyASCIIObject \
--whitelist-type PyUnicodeObject \
--whitelist-type PyCompactUnicodeObject \
--whitelist-type PyTupleObject \
--whitelist-type PyListObject \
--whitelist-type PyLongObject \
--whitelist-type PyFloatObject \
--whitelist-type PyDictObject \
--whitelist-type PyDictKeysObject \
--whitelist-type PyObject \
--whitelist-type PyTypeObject \
--whitelist-type PyHeapTypeObject \
-- -I . -I ./Include -I ./Include/internal
"""
)
if ret:
return ret
with open(os.path.join("src", "python_bindings", version.replace(".", "_") + ".rs"), "w") as o:
o.write(f"// Generated bindings for python {version}\n")
o.write("#![allow(dead_code)]\n")
o.write("#![allow(non_upper_case_globals)]\n")
o.write("#![allow(non_camel_case_types)]\n")
o.write("#![allow(non_snake_case)]\n")
o.write("#![allow(clippy::useless_transmute)]\n")
o.write("#![allow(clippy::default_trait_access)]\n")
o.write("#![allow(clippy::cast_lossless)]\n")
o.write("#![allow(clippy::trivially_copy_pass_by_ref)]\n")
o.write("#![allow(clippy::upper_case_acronyms)]\n")
o.write("#![allow(clippy::too_many_arguments)]\n\n")
o.write(open(os.path.join(cpython_path, "bindgen_output.rs")).read())
if __name__ == "__main__":
if sys.platform.startswith("win"):
default_cpython_path = os.path.join(os.getenv("userprofile"), "code", "cpython")
else:
default_cpython_path = os.path.join(os.getenv("HOME"), "code", "cpython")
parser = argparse.ArgumentParser(
description="runs bindgen on cpython version",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--cpython",
type=str,
default=default_cpython_path,
dest="cpython",
help="path to cpython repo",
)
parser.add_argument(
"--configure",
help="Run configure script prior to generating bindings",
action="store_true",
)
parser.add_argument("--pyruntime", help="generate offsets for pyruntime", action="store_true")
parser.add_argument("--build", help="Build python for this version", action="store_true")
parser.add_argument("--all", help="Build all versions", action="store_true")
parser.add_argument("versions", type=str, nargs="*", help="versions to extract")
args = parser.parse_args()
if not os.path.isdir(args.cpython):
print(f"Directory '{args.cpython}' doesn't exist!")
print("Pass a valid cpython path in with --cpython <pathname>")
sys.exit(1)
if args.all:
versions = [
"v3.8.0b4",
"v3.7.0",
"v3.6.6",
"v3.5.5",
"v3.4.8",
"v3.3.7",
"v3.2.6",
"v2.7.15",
]
else:
versions = args.versions
if not versions:
print("You must specify versions of cpython to generate bindings for, or --all\n")
parser.print_help()
for version in versions:
if args.build:
if build_python(args.cpython, version):
print("Failed to build python")
elif args.pyruntime:
calculate_pyruntime_offsets(args.cpython, version, configure=args.configure)
else:
if extract_bindings(args.cpython, version, configure=args.configure):
print("Failed to generate bindings")