from __future__ import annotations
import itertools
import os
import shlex
import sys
import tempfile
from pathlib import Path
from typing import IO, Any, BinaryIO, cast
import click
from build import BuildBackendException
from click.utils import LazyFile, safecall
from pip._internal.req import InstallRequirement
from pip._internal.req.constructors import install_req_from_line
from pip._internal.utils.misc import redact_auth_from_url
from .._compat import parse_requirements
from ..build import build_project_metadata
from ..cache import DependencyCache
from ..exceptions import NoCandidateFound, PipToolsError
from ..logging import log
from ..repositories import LocalRequirementsRepository, PyPIRepository
from ..repositories.base import BaseRepository
from ..resolver import BacktrackingResolver, LegacyResolver
from ..utils import dedup, drop_extras, is_pinned_requirement, key_from_ireq
from ..writer import OutputWriter
from . import options
from .options import BuildTargetT
DEFAULT_REQUIREMENTS_FILES = (
"requirements.in",
"setup.py",
"pyproject.toml",
"setup.cfg",
)
DEFAULT_REQUIREMENTS_FILE = "requirements.in"
DEFAULT_REQUIREMENTS_OUTPUT_FILE = "requirements.txt"
METADATA_FILENAMES = frozenset({"setup.py", "setup.cfg", "pyproject.toml"})
def _determine_linesep(
strategy: str = "preserve", filenames: tuple[str, ...] = ()
) -> str:
if strategy == "preserve":
for fname in filenames:
try:
with open(fname, "rb") as existing_file:
existing_text = existing_file.read()
except FileNotFoundError:
continue
if b"\r\n" in existing_text:
strategy = "CRLF"
break
elif b"\n" in existing_text:
strategy = "LF"
break
return {
"native": os.linesep,
"LF": "\n",
"CRLF": "\r\n",
"preserve": "\n",
}[strategy]
@click.command(
name="pip-compile",
context_settings={"help_option_names": options.help_option_names},
)
@click.pass_context
@options.version
@options.color
@options.verbose
@options.quiet
@options.dry_run
@options.pre
@options.rebuild
@options.extra
@options.all_extras
@options.find_links
@options.index_url
@options.no_index
@options.extra_index_url
@options.cert
@options.client_cert
@options.trusted_host
@options.header
@options.emit_trusted_host
@options.annotate
@options.annotation_style
@options.upgrade
@options.upgrade_package
@options.output_file
@options.newline
@options.allow_unsafe
@options.strip_extras
@options.generate_hashes
@options.reuse_hashes
@options.max_rounds
@options.src_files
@options.build_isolation
@options.emit_find_links
@options.cache_dir
@options.pip_args
@options.resolver
@options.emit_index_url
@options.emit_options
@options.unsafe_package
@options.config
@options.no_config
@options.constraint
@options.build_deps_for
@options.all_build_deps
@options.only_build_deps
def cli(
ctx: click.Context,
color: bool | None,
verbose: int,
quiet: int,
dry_run: bool,
pre: bool,
rebuild: bool,
extras: tuple[str, ...],
all_extras: bool,
find_links: tuple[str, ...],
index_url: str,
no_index: bool,
extra_index_url: tuple[str, ...],
cert: str | None,
client_cert: str | None,
trusted_host: tuple[str, ...],
header: bool,
emit_trusted_host: bool,
annotate: bool,
annotation_style: str,
upgrade: bool,
upgrade_packages: tuple[str, ...],
output_file: LazyFile | IO[Any] | None,
newline: str,
allow_unsafe: bool,
strip_extras: bool | None,
generate_hashes: bool,
reuse_hashes: bool,
src_files: tuple[str, ...],
max_rounds: int,
build_isolation: bool,
emit_find_links: bool,
cache_dir: str,
pip_args_str: str | None,
resolver_name: str,
emit_index_url: bool,
emit_options: bool,
unsafe_package: tuple[str, ...],
config: Path | None,
no_config: bool,
constraint: tuple[str, ...],
build_deps_targets: tuple[BuildTargetT, ...],
all_build_deps: bool,
only_build_deps: bool,
) -> None:
if color is not None:
ctx.color = color
log.verbosity = verbose - quiet
if not src_files and ctx.default_map and "src_files" in ctx.default_map:
src_files = ctx.default_map["src_files"]
if all_build_deps and build_deps_targets:
raise click.BadParameter(
"--build-deps-for has no effect when used with --all-build-deps"
)
elif all_build_deps:
build_deps_targets = options.ALL_BUILD_TARGETS
if only_build_deps and not build_deps_targets:
raise click.BadParameter(
"--only-build-deps requires either --build-deps-for or --all-build-deps"
)
if only_build_deps and (extras or all_extras):
raise click.BadParameter(
"--only-build-deps cannot be used with any of --extra, --all-extras"
)
if len(src_files) == 0:
for file_path in DEFAULT_REQUIREMENTS_FILES:
if os.path.exists(file_path):
src_files = (file_path,)
break
else:
raise click.BadParameter(
(
"If you do not specify an input file, the default is one of: {}"
).format(", ".join(DEFAULT_REQUIREMENTS_FILES))
)
if all_extras and extras:
msg = "--extra has no effect when used with --all-extras"
raise click.BadParameter(msg)
if not output_file:
if src_files == ("-",):
raise click.BadParameter("--output-file is required if input is from stdin")
elif os.path.basename(src_files[0]) in METADATA_FILENAMES:
file_name = os.path.join(
os.path.dirname(src_files[0]), DEFAULT_REQUIREMENTS_OUTPUT_FILE
)
elif len(src_files) > 1:
raise click.BadParameter(
"--output-file is required if two or more input files are given."
)
else:
base_name = src_files[0].rsplit(".", 1)[0]
file_name = base_name + ".txt"
output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True)
assert output_file is not None
if isinstance(output_file, LazyFile): ctx.call_on_close(safecall(output_file.close_intelligently))
if output_file.name != "-" and output_file.name in src_files:
raise click.BadArgumentUsage(
f"input and output filenames must not be matched: {output_file.name}"
)
if config:
log.debug(f"Using pip-tools configuration defaults found in '{config !s}'.")
if resolver_name == "legacy":
log.warning(
"WARNING: the legacy dependency resolver is deprecated and will be removed"
" in future versions of pip-tools."
)
right_args = shlex.split(pip_args_str or "")
pip_args = []
for link in find_links:
pip_args.extend(["-f", link])
if index_url:
pip_args.extend(["-i", index_url])
if no_index:
pip_args.extend(["--no-index"])
for extra_index in extra_index_url:
pip_args.extend(["--extra-index-url", extra_index])
if cert:
pip_args.extend(["--cert", cert])
if client_cert:
pip_args.extend(["--client-cert", client_cert])
if pre:
pip_args.extend(["--pre"])
for host in trusted_host:
pip_args.extend(["--trusted-host", host])
if not build_isolation:
pip_args.append("--no-build-isolation")
if resolver_name == "legacy":
pip_args.extend(["--use-deprecated", "legacy-resolver"])
if resolver_name == "backtracking" and cache_dir:
pip_args.extend(["--cache-dir", cache_dir])
pip_args.extend(right_args)
repository: BaseRepository
repository = PyPIRepository(pip_args, cache_dir=cache_dir)
upgrade_reqs_gen = (install_req_from_line(pkg) for pkg in upgrade_packages)
upgrade_install_reqs = {
key_from_ireq(install_req): install_req for install_req in upgrade_reqs_gen
}
existing_pins = {}
output_file_exists = os.path.exists(output_file.name)
if not upgrade and output_file_exists:
output_file_is_empty = os.path.getsize(output_file.name) == 0
if upgrade_install_reqs and output_file_is_empty:
log.warning(
f"WARNING: the output file {output_file.name} exists but is empty. "
"Pip-tools cannot upgrade only specific packages (using -P/--upgrade-package) "
"without an existing pin file to provide constraints. "
"This often occurs if you redirect standard output to your output file, "
"as any existing content is truncated."
)
tmp_repository = PyPIRepository(pip_args, cache_dir=cache_dir)
ireqs = parse_requirements(
output_file.name,
finder=tmp_repository.finder,
session=tmp_repository.session,
options=tmp_repository.options,
)
for ireq in filter(is_pinned_requirement, ireqs):
key = key_from_ireq(ireq)
if key not in upgrade_install_reqs:
existing_pins[key] = ireq
repository = LocalRequirementsRepository(
existing_pins, repository, reuse_hashes=reuse_hashes
)
constraints: list[InstallRequirement] = []
setup_file_found = False
for src_file in src_files:
is_setup_file = os.path.basename(src_file) in METADATA_FILENAMES
if not is_setup_file and build_deps_targets:
msg = (
"--build-deps-for and --all-build-deps can be used only with the "
"setup.py, setup.cfg and pyproject.toml specs."
)
raise click.BadParameter(msg)
if src_file == "-":
tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False)
tmpfile.write(sys.stdin.read())
comes_from = "-r -"
tmpfile.flush()
reqs = list(
parse_requirements(
tmpfile.name,
finder=repository.finder,
session=repository.session,
options=repository.options,
)
)
for req in reqs:
req.comes_from = comes_from
constraints.extend(reqs)
elif is_setup_file:
setup_file_found = True
try:
metadata = build_project_metadata(
src_file=Path(src_file),
build_targets=build_deps_targets,
isolated=build_isolation,
quiet=log.verbosity <= 0,
)
except BuildBackendException as e:
log.error(str(e))
log.error(f"Failed to parse {os.path.abspath(src_file)}")
sys.exit(2)
if not only_build_deps:
constraints.extend(metadata.requirements)
if all_extras:
extras += metadata.extras
if build_deps_targets:
constraints.extend(metadata.build_requirements)
else:
constraints.extend(
parse_requirements(
src_file,
finder=repository.finder,
session=repository.session,
options=repository.options,
)
)
for filename in constraint:
constraints.extend(
parse_requirements(
filename,
constraint=True,
finder=repository.finder,
options=repository.options,
session=repository.session,
)
)
if upgrade_packages:
constraints_file = tempfile.NamedTemporaryFile(mode="wt", delete=False)
constraints_file.write("\n".join(upgrade_packages))
constraints_file.flush()
try:
reqs = list(
parse_requirements(
constraints_file.name,
finder=repository.finder,
session=repository.session,
options=repository.options,
constraint=True,
)
)
finally:
constraints_file.close()
os.unlink(constraints_file.name)
for req in reqs:
req.comes_from = None
constraints.extend(reqs)
extras = tuple(itertools.chain.from_iterable(ex.split(",") for ex in extras))
if extras and not setup_file_found:
msg = "--extra has effect only with setup.py and PEP-517 input formats"
raise click.BadParameter(msg)
primary_packages = {
key_from_ireq(ireq) for ireq in constraints if not ireq.constraint
}
constraints.extend(
ireq for key, ireq in upgrade_install_reqs.items() if key in primary_packages
)
constraints = [req for req in constraints if req.match_markers(extras)]
for req in constraints:
drop_extras(req)
if repository.finder.index_urls:
log.debug("Using indexes:")
with log.indentation():
for index_url in dedup(repository.finder.index_urls):
log.debug(redact_auth_from_url(index_url))
else:
log.debug("Ignoring indexes.")
if repository.finder.find_links:
log.debug("")
log.debug("Using links:")
with log.indentation():
for find_link in dedup(repository.finder.find_links):
log.debug(redact_auth_from_url(find_link))
resolver_cls = LegacyResolver if resolver_name == "legacy" else BacktrackingResolver
try:
resolver = resolver_cls(
constraints=constraints,
existing_constraints=existing_pins,
repository=repository,
prereleases=repository.finder.allow_all_prereleases or pre,
cache=DependencyCache(cache_dir),
clear_caches=rebuild,
allow_unsafe=allow_unsafe,
unsafe_packages=set(unsafe_package),
)
results = resolver.resolve(max_rounds=max_rounds)
hashes = resolver.resolve_hashes(results) if generate_hashes else None
except NoCandidateFound as e:
if resolver_cls == LegacyResolver: log.error(
"Using legacy resolver. "
"Consider using backtracking resolver with "
"`--resolver=backtracking`."
)
log.error(str(e))
sys.exit(2)
except PipToolsError as e:
log.error(str(e))
sys.exit(2)
log.debug("")
linesep = _determine_linesep(
strategy=newline, filenames=(output_file.name, *src_files)
)
if strip_extras is None:
strip_extras = False
log.warning(
"WARNING: --strip-extras is becoming the default "
"in version 8.0.0. To silence this warning, "
"either use --strip-extras to opt into the new default "
"or use --no-strip-extras to retain the existing behavior."
)
writer = OutputWriter(
cast(BinaryIO, output_file),
click_ctx=ctx,
dry_run=dry_run,
emit_header=header,
emit_index_url=emit_index_url,
emit_trusted_host=emit_trusted_host,
annotate=annotate,
annotation_style=annotation_style,
strip_extras=strip_extras,
generate_hashes=generate_hashes,
default_index_url=repository.DEFAULT_INDEX_URL,
index_urls=repository.finder.index_urls,
trusted_hosts=repository.finder.trusted_hosts,
format_control=repository.finder.format_control,
linesep=linesep,
allow_unsafe=allow_unsafe,
find_links=repository.finder.find_links,
emit_find_links=emit_find_links,
emit_options=emit_options,
)
writer.write(
results=results,
unsafe_packages=resolver.unsafe_packages,
unsafe_requirements=resolver.unsafe_constraints,
markers={
key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers
},
hashes=hashes,
)
if dry_run:
log.info("Dry-run, so nothing updated.")