py-spy 0.1.10

Sampling profiler for Python programs 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

""" Scripts to generate bindings of different python interperator versions

Requires bindgen to be installed (cargo install bindgen), and probably needs a nightly
compiler with rustfmt-nightly.

Also requires a git repo of cpython to be checked out somewhere. As a hack, this can
also build different versions of cpython for testing out
"""
import argparse
import os
import sys


def build_python(cpython_path, version):
    # TODO: probably easier to use pyenv for this?
    print("Compiling python %s from repo at %s" % (version, cpython_path))
    install_path = os.path.join(cpython_path, version)

    ret = os.system(f"""
        cd {cpython_path}
        git checkout {version}

        # build in a subdirectory
        mkdir -p build_{version}
        cd build_{version}
        ../configure prefix={install_path}
        make
        make install
    """)
    if ret:
        return ret

    # also install setuptools_rust/wheel here for building packages
    pip = os.path.join(install_path, "bin", "pip3" if version.startswith("v3") else "pip")
    return os.system(f"{pip} install setuptools_rust wheel")


def extract_bindings(cpython_path, version, configure=False):
    print("Generating bindings for python %s from repo at %s" % (version, cpython_path))
    ret = os.system(f"""
        cd {cpython_path}
        git checkout {version}

        # need to run configure on the current branch to generate pyconfig.h sometimes
        {("./configure prefix=" + os.path.join(cpython_path, version)) if configure else ""}

        cat include/Python.h > bindgen_input.h
        cat include/frameobject.h >> bindgen_input.h

        bindgen  bindgen_input.h -o bindgen_output.rs \
            --with-derive-default \
            --no-layout-tests --no-doc-comments \
            --whitelist-type PyInterpreterState \
            --whitelist-type PyFrameObject \
            --whitelist-type PyThreadState \
            --whitelist-type PyCodeObject \
            --whitelist-type PyVarObject \
            --whitelist-type PyBytesObject \
            --whitelist-type PyASCIIObject \
            --whitelist-type PyUnicodeObject \
            --whitelist-type PyCompactUnicodeObject \
            --whitelist-type PyStringObject \
             -- -I . -I ./Include
    """)
    if ret:
        return ret

    # write the file out to the appropiate place, disabling some warnings
    with open(os.path.join("src", "python_bindings", version.replace(".", "_") + ".rs"), "w") as o:
        o.write(f"// Generated bindings for python {version}\n")
        o.write("#![allow(dead_code)]\n")
        o.write("#![allow(non_upper_case_globals)]\n")
        o.write("#![allow(non_camel_case_types)]\n")
        o.write("#![allow(non_snake_case)]\n")
        o.write("#![cfg_attr(feature = \"cargo-clippy\", allow(useless_transmute))]\n")
        o.write("#![cfg_attr(feature = \"cargo-clippy\", allow(default_trait_access))]\n")
        o.write("#![cfg_attr(feature = \"cargo-clippy\", allow(cast_lossless))]\n")
        o.write("#![cfg_attr(feature = \"cargo-clippy\", allow(trivially_copy_pass_by_ref))]\n\n")
        o.write(open(os.path.join(cpython_path, "bindgen_output.rs")).read())


if __name__ == "__main__":
    default_cpython_path = os.path.join(os.getenv("HOME"), "code", "cpython")

    parser = argparse.ArgumentParser(description="runs bindgen on cpython version",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--cpython", type=str, default=default_cpython_path,
                        dest="cpython", help="path to cpython repo")
    parser.add_argument("--configure",
                        help="Run configure script prior to generating bindings",
                        action="store_true")
    parser.add_argument("--build",
                        help="Build python for this version",
                        action="store_true")
    parser.add_argument("--all",
                        help="Build all versions",
                        action="store_true")

    parser.add_argument("versions", type=str, nargs='*', help='versions to extract')

    args = parser.parse_args()

    if not os.path.isdir(args.cpython):
        print(f"Directory '{args.cpython}' doesn't exist!")
        print("Pass a valid cpython path in with --cpython <pathname>")
        sys.exit(1)

    if args.all:
        versions = ['v3.7.0', 'v3.6.6', 'v3.5.5', 'v3.4.8', 'v3.3.7', 'v3.2.6', 'v2.7.15']
    else:
        versions = args.versions
        if not versions:
            print("You must specify versions of cpython to generate bindings for, or --all\n")
            parser.print_help()

    for version in versions:
        if args.build:
            # todo: this probably shoudl be a separate script
            if build_python(args.cpython, version):
                print("Failed to build python")
        else:
            if extract_bindings(args.cpython, version, configure=args.configure):
                print("Failed to generate bindings")