import re
import sys
import venv
import json
import shutil
import argparse
import tempfile
from plumbum import local
from plumbum.cmd import tar, which, unzip, fpm, git, \
mkdir, wget, rm, sed, find, python3, apt, dpkg, cp
import os.path
import logging
import importlib.util
def get_pkg_name(pkg_source_path):
with local.cwd(pkg_source_path):
return python3('setup.py', '--name').strip().split()[-1]
def get_pkg_dependencies(pkg_source_path):
setup_py_path = os.path.join(pkg_source_path, 'setup.py')
with local.cwd(pkg_source_path):
lines = python3('setup.py', 'install', '--verbose', '--dry-run',
'--user', '--skip-build', '--no-compile')
deps = {}
for l in lines.split('\n'):
if l.startswith('Searching for'):
name_ver = l.split(' ')[-1].split('==')
deps.update(
{name_ver[0]: name_ver[1] if len(name_ver) == 2 else None}
)
return deps
def load_config(path):
return json.load(open(path))
def _get_fpm_args(pkg_name, build_conf):
common_fpm_args = build_conf.get('fpm-args', [])
pkg_fpm_args = build_conf['packages'][pkg_name].get('fpm-args', [])
version = build_conf['packages'][pkg_name].get('version')
maintainer = build_conf['packages'][pkg_name].get('maintainer')
return common_fpm_args + \
pkg_fpm_args + \
(['-v', version] if version else []) + \
['-p', build_conf["output_path"]] + \
['-m', maintainer]
def clean_build(pkg_source_path):
python3(os.path.join(pkg_source_path, 'setup.py'), 'clean', '--all')
find(pkg_source_path, '-name', '*.egg-info', '-exec', 'rm', '-fr', '{}', '+')
find(pkg_source_path, '-name', '*.eggs', '-exec', 'rm', '-fr', '{}', '+')
find(pkg_source_path, '-name', '*.pyc', '-exec', 'rm', '{}', '+')
rm('-rf', os.path.join(pkg_source_path, 'build'))
rm('-rf', os.path.join(pkg_source_path, 'dist'))
rm('-rf', os.path.join(pkg_source_path, '.eggs'))
logging.warning('"%s" cleaned up', pkg_source_path)
def build_pkg_pypi(pkg_name, build_conf):
build_args = _get_fpm_args(pkg_name, build_conf)
build_args += build_conf['fpm-python-args']
build_args = with_python_preinst_prerm_hooks(pkg_name, build_args)
fpm(build_args + [pkg_name])
def build_pkg_filesystem(pkg_name, pkg_source_path, build_conf):
build_args = _get_fpm_args(pkg_name, build_conf)
build_args += build_conf['fpm-python-args']
build_args = with_python_preinst_prerm_hooks(pkg_name, build_args)
fpm(build_args + [pkg_source_path])
def with_python_preinst_prerm_hooks(pkg_name, build_args):
name = 'python3-' + pkg_name.lower()
if '-n' in build_args:
idx = build_args.index('-n')
name = build_args[idx + 1]
after_install_path = None
if "--after-install" in build_args:
idx = build_args.index("--after-install")
after_install_path = build_args[idx + 1]
del build_args[idx]
del build_args[idx]
build_args += [
"--after-install",
render_python_hook("postinst", name, after_install_path),
]
before_remove_path = None
if "--before-remove" in build_args:
idx = build_args.index("--before-remove")
before_remove_path = build_args[idx + 1]
del build_args[idx]
del build_args[idx]
build_args += [
"--before-remove",
render_python_hook("prerm", name, before_remove_path),
]
return build_args
def render_python_hook(template_name, pkg_name, exist_hook_name=None):
template_content = open(template_name).read()
file_content = template_content.format(package_name=pkg_name)
tmp_dir = tempfile.mkdtemp()
if exist_hook_name:
exist_hook_content = open(exist_hook_name).read()
file_content = exist_hook_content + '\n'*3 + file_content
output_file_path = os.path.join(tmp_dir, exist_hook_name)
else:
output_file_path = os.path.join(tmp_dir, template_name)
fd = open(output_file_path, 'a+')
fd.write(file_content)
fd.close()
return output_file_path
def _build_pkg_pbc(pkg_name, build_conf):
pkg_conf = build_conf['packages'][pkg_name]
version = pkg_conf['version']
with local.cwd('/tmp'):
if not local.path(local.cwd / 'pbc').exists():
git('clone', 'http://repo.or.cz/r/pbc.git')
with local.cwd(local.cwd / 'pbc'):
git('checkout', '656ae0c90e120eacd3dc0d76dbc9504f8aca4ba8')
local['dpkg-buildpackage']('-uc', '-us')
pbc_lib_deb = 'libpbc0_{}_amd64.deb'.format(version)
pbc_dev_deb = 'libpbc-dev_{}_amd64.deb'.format(version)
dpkg('-i', pbc_lib_deb)
dpkg('-i', pbc_dev_deb)
cp(pbc_lib_deb, build_conf['output_path'])
cp(pbc_dev_deb, build_conf['output_path'])
def build_deps(deps, build_conf):
for name, version in deps.items():
dep_conf = build_conf["packages"].get(name)
if dep_conf is None:
logging.warning('Skip building "%s", not found config', name)
continue
extra_deps = dict.fromkeys(dep_conf.get("requires", []), None)
build_deps(extra_deps, build_conf)
logging.info('Build "%s"', name)
if dep_conf["origin"] == 'pypi':
build_pkg_pypi(name, build_conf)
elif dep_conf["origin"] == 'custom':
custom_builder_name = '_build_pkg_{}'.format(name.lower().replace('-', '_'))
custom_builder = globals().get(custom_builder_name)
if not custom_builder:
raise Exception('Cannot find custom builder "{}"'.format(custom_builder_name))
custom_builder(name, build_conf)
def main(pkg_name, pkg_source_path, output_path):
clean_build(pkg_source_path)
build_conf = load_config('deb-build.json')
build_conf['output_path'] = output_path
if pkg_name not in build_conf["packages"]:
logging.error('Config was not found for "%s"', pkg_name)
sys.exit(1)
deps = {}
extra_deps = build_conf["packages"][pkg_name].get("requires", [])
deps.update(dict.fromkeys(extra_deps, None))
build_deps(deps, build_conf)
build_pkg_filesystem(pkg_name, pkg_source_path, build_conf)
clean_build(pkg_source_path)
if __name__ == '__main__':
FORMAT = '%(asctime)-15s [%(name)-15s] %(levelname)-8s %(message)s'
logging.basicConfig(format=FORMAT, level=logging.DEBUG)
parser = argparse.ArgumentParser()
parser.add_argument('pkg_name')
parser.add_argument('pkg_source_path')
parser.add_argument('output_path')
args = vars(parser.parse_args())
main(
args['pkg_name'],
args['pkg_source_path'],
args['output_path']
)