import argparse
import logging
import os
import pathlib
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
import urllib.request
import scripthash
_THIS_DIR = pathlib.Path(__file__).resolve().parent
_SRC_ROOT = _THIS_DIR.parents[1]
_CHECKOUT_SRC_ROOT_SUBDIR = '.3pp/chromium'
def parse_args():
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
subparser = subparsers.add_parser(
'latest', help='Prints the version as $LATEST.$RUNTIME_DEPS_HASH')
subparser.set_defaults(action='latest')
subparser = subparsers.add_parser(
'checkout', help='Copies files into the workdir used by docker')
subparser.add_argument('checkout_dir')
subparser.add_argument('--version', help='Output from "latest"')
subparser.set_defaults(action='checkout')
subparser = subparsers.add_parser(
'install',
help=('Run from workdir, inside docker container. '
'Builds & copies outputs into |output_prefix| directory'))
subparser.add_argument('output_prefix',
help='The path to install the compiled package to.')
subparser.add_argument('deps_prefix',
help='The path to a directory containing all deps.')
subparser.add_argument('--version', help='Output from "latest"')
subparser.add_argument('--checkout-dir', help='Directory to use as CWD')
subparser.set_defaults(action='install')
subparser = subparsers.add_parser(
'local-test', help='Run latest / checkout / install locally')
subparser.add_argument('--checkout-dir',
default='3pp_workdir',
help='Workdir to use')
subparser.add_argument('--output-prefix',
default='3pp_out',
help='Directory for final artifacts')
subparser.set_defaults(action='local-test')
args = parser.parse_args()
if not hasattr(args, 'action'):
parser.print_help()
sys.exit(1)
if hasattr(args, 'version'):
args.version = args.version or os.environ.get('_3PP_VERSION')
if not args.version:
parser.error('Must set --version or _3PP_VERSION')
if hasattr(args, 'output_prefix') and args.output_prefix:
args.output_prefix = os.path.abspath(args.output_prefix)
if hasattr(args, 'checkout_dir') and args.checkout_dir:
args.checkout_dir = os.path.abspath(args.checkout_dir)
if args.action == 'checkout':
os.makedirs(args.checkout_dir, exist_ok=True)
if args.action == 'install':
if args.checkout_dir:
logging.info('Setting CWD=%s', args.checkout_dir)
os.chdir(args.checkout_dir)
if not os.path.exists(_CHECKOUT_SRC_ROOT_SUBDIR):
parser.error(f'Does not exist: {_CHECKOUT_SRC_ROOT_SUBDIR}.'
f' Use --checkout-dir?')
os.makedirs(args.output_prefix, exist_ok=True)
return args
def path_within_checkout(subpath):
return os.path.abspath(os.path.join(_CHECKOUT_SRC_ROOT_SUBDIR, subpath))
def _all_files(path):
if os.path.isfile(path):
return [path]
assert os.path.isdir(path), 'Not a file or dir: ' + path
all_paths = pathlib.Path(path).glob('**/*')
return [str(f) for f in all_paths if f.is_file()]
def _resolve_runtime_deps(runtime_deps):
ret = []
for p in runtime_deps:
if p.startswith('//'):
ret.append(os.path.relpath(str(_SRC_ROOT / p[2:])))
elif os.path.isabs(p):
ret.append(os.path.relpath(p))
else:
ret.append(p)
return ret
def copy_runtime_deps(checkout_dir, runtime_deps):
dest_dir = os.path.join(checkout_dir, _CHECKOUT_SRC_ROOT_SUBDIR)
for src_path in _resolve_runtime_deps(runtime_deps):
relpath = os.path.relpath(src_path, _SRC_ROOT)
dest_path = os.path.join(dest_dir, relpath)
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
if os.path.isfile(src_path):
shutil.copy(src_path, dest_path)
else:
shutil.copytree(src_path,
dest_path,
ignore=shutil.ignore_patterns('.*', '__pycache__'))
logging.info('Runtime deps:')
sys.stderr.write('\n'.join(_all_files(checkout_dir)) + '\n')
def download_file(url, dest):
logging.info('Downloading %s', url)
with urllib.request.urlopen(url) as r:
with open(dest, 'wb') as f:
shutil.copyfileobj(r, f)
def extract_tar(path, dest):
logging.info('Extracting %s to %s', path, dest)
with tarfile.open(path) as f:
f.extractall(dest)
def run_cmd(cmd, check=True, *args, **kwargs):
logging.info('Running: %s', shlex.join(cmd))
return subprocess.run(cmd, check=check, *args, **kwargs)
def apply_patches(patches_dir, checkout_dir):
for path in sorted(pathlib.Path(patches_dir).glob('*.patch')):
cmd = ['git', 'apply', '-v', str(path)]
run_cmd(cmd, cwd=checkout_dir)
def main(*, do_latest, do_install, runtime_deps):
logging.basicConfig(
level=logging.DEBUG,
format='%(levelname).1s %(relativeCreated)6d %(message)s')
args = parse_args()
runtime_deps = [str(_THIS_DIR)] + runtime_deps
if args.action == 'local-test':
logging.warning('Will use work dir: %s', args.checkout_dir)
logging.warning('Will use output dir: %s', args.output_prefix)
if os.path.exists(args.checkout_dir) and os.listdir(args.checkout_dir):
logging.warning(
'*** Work dir not empty. This often causes failures. ***')
time.sleep(4)
prog = os.path.abspath(sys.argv[0])
cmd = [prog, 'latest']
version = run_cmd(cmd, stdout=subprocess.PIPE, text=True).stdout
os.environ['_3PP_VERSION'] = version
checkout_dir = args.checkout_dir
run_cmd([prog, 'checkout', checkout_dir])
run_cmd([prog, 'install', args.output_prefix, 'UNUSED-DEPS-DIR'],
cwd=checkout_dir)
logging.warning('Local test complete.')
return
if args.action == 'latest':
version = do_latest()
assert version, 'do_latest() returned ' + repr(version)
extra_paths = []
for p in _resolve_runtime_deps(runtime_deps):
extra_paths += _all_files(p)
deps_hash = scripthash.compute(extra_paths=extra_paths)
print(f'{version}.{deps_hash}')
return
args.version = args.version.rsplit('.', 1)[0]
if args.action == 'checkout':
copy_runtime_deps(args.checkout_dir, runtime_deps)
return
assert args.action == 'install'
do_install(args)
prefix_len = len(args.output_prefix) + 1
logging.info(
'Contents of %s: \n%s\n', args.output_prefix,
'\n'.join(p[prefix_len:] for p in _all_files(args.output_prefix)))