更改enroll命名,添加了注释,向get_error_msg中添加了一些错误代码

This commit is contained in:
ygm1881
2022-05-05 22:59:35 +08:00
parent 51b5e374a3
commit ece69eaf57
4637 changed files with 7699 additions and 608140 deletions
@@ -93,9 +93,7 @@ class BuildEnvironment:
self._site_dir = os.path.join(temp_dir.path, "site")
if not os.path.exists(self._site_dir):
os.mkdir(self._site_dir)
with open(
os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
) as fp:
with open(os.path.join(self._site_dir, "sitecustomize.py"), "w") as fp:
fp.write(
textwrap.dedent(
"""
@@ -189,8 +187,7 @@ class BuildEnvironment:
finder: "PackageFinder",
requirements: Iterable[str],
prefix_as_string: str,
*,
kind: str,
message: str,
) -> None:
prefix = self._prefixes[prefix_as_string]
assert not prefix.setup
@@ -198,13 +195,20 @@ class BuildEnvironment:
if not requirements:
return
with contextlib.ExitStack() as ctx:
pip_runnable = ctx.enter_context(_create_standalone_pip())
# TODO: Remove this block when dropping 3.6 support. Python 3.6
# lacks importlib.resources and pep517 has issues loading files in
# a zip, so we fallback to the "old" method by adding the current
# pip directory to the child process's sys.path.
if sys.version_info < (3, 7):
pip_runnable = os.path.dirname(pip_location)
else:
pip_runnable = ctx.enter_context(_create_standalone_pip())
self._install_requirements(
pip_runnable,
finder,
requirements,
prefix,
kind=kind,
message,
)
@staticmethod
@@ -213,8 +217,7 @@ class BuildEnvironment:
finder: "PackageFinder",
requirements: Iterable[str],
prefix: _Prefix,
*,
kind: str,
message: str,
) -> None:
args: List[str] = [
sys.executable,
@@ -256,13 +259,8 @@ class BuildEnvironment:
args.append("--")
args.extend(requirements)
extra_environ = {"_PIP_STANDALONE_CERT": where()}
with open_spinner(f"Installing {kind}") as spinner:
call_subprocess(
args,
command_desc=f"pip subprocess to install {kind}",
spinner=spinner,
extra_environ=extra_environ,
)
with open_spinner(message) as spinner:
call_subprocess(args, spinner=spinner, extra_environ=extra_environ)
class NoOpBuildEnvironment(BuildEnvironment):
@@ -290,7 +288,6 @@ class NoOpBuildEnvironment(BuildEnvironment):
finder: "PackageFinder",
requirements: Iterable[str],
prefix_as_string: str,
*,
kind: str,
message: str,
) -> None:
raise NotImplementedError()
@@ -59,14 +59,6 @@ def autocomplete() -> None:
print(dist)
sys.exit(1)
should_list_installables = (
not current.startswith("-") and subcommand_name == "install"
)
if should_list_installables:
for path in auto_complete_paths(current, "path"):
print(path)
sys.exit(1)
subcommand = create_command(subcommand_name)
for opt in subcommand.parser.option_list_all:
@@ -146,7 +138,7 @@ def auto_complete_paths(current: str, completion_type: str) -> Iterable[str]:
starting with ``current``.
:param current: The word to be completed
:param completion_type: path completion type(``file``, ``path`` or ``dir``)
:param completion_type: path completion type(`file`, `path` or `dir`)i
:return: A generator of regular files and/or directories
"""
directory, filename = os.path.split(current)
@@ -10,8 +10,6 @@ import traceback
from optparse import Values
from typing import Any, Callable, List, Optional, Tuple
from pip._vendor.rich import traceback as rich_traceback
from pip._internal.cli import cmdoptions
from pip._internal.cli.command_context import CommandContextMixIn
from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
@@ -24,7 +22,6 @@ from pip._internal.cli.status_codes import (
from pip._internal.exceptions import (
BadCommand,
CommandError,
DiagnosticPipError,
InstallationError,
NetworkConnectionError,
PreviousBuildDirError,
@@ -167,11 +164,6 @@ class Command(CommandContextMixIn):
status = run_func(*args)
assert isinstance(status, int)
return status
except DiagnosticPipError as exc:
logger.error("[present-diagnostic] %s", exc)
logger.debug("Exception information:", exc_info=True)
return ERROR
except PreviousBuildDirError as exc:
logger.critical(str(exc))
logger.debug("Exception information:", exc_info=True)
@@ -217,7 +209,6 @@ class Command(CommandContextMixIn):
run = intercepts_unhandled_exc(self.run)
else:
run = self.run
rich_traceback.install(show_locals=True)
return run(options, args)
finally:
self.handle_pip_version_check(options)
@@ -10,9 +10,9 @@ pass on state. To be consistent, all options will follow this design.
# The following comment should be removed at some point in the future.
# mypy: strict-optional=False
import logging
import os
import textwrap
import warnings
from functools import partial
from optparse import SUPPRESS_HELP, Option, OptionGroup, OptionParser, Values
from textwrap import dedent
@@ -30,8 +30,6 @@ from pip._internal.models.target_python import TargetPython
from pip._internal.utils.hashes import STRONG_HASHES
from pip._internal.utils.misc import strtobool
logger = logging.getLogger(__name__)
def raise_option_error(parser: OptionParser, option: Option, msg: str) -> None:
"""
@@ -78,9 +76,10 @@ def check_install_build_global(
if any(map(getname, names)):
control = options.format_control
control.disallow_binaries()
logger.warning(
warnings.warn(
"Disabling all use of wheels due to the use of --build-option "
"/ --global-option / --install-option.",
stacklevel=2,
)
@@ -178,15 +177,13 @@ isolated_mode: Callable[..., Option] = partial(
require_virtualenv: Callable[..., Option] = partial(
Option,
# Run only if inside a virtualenv, bail if not.
"--require-virtualenv",
"--require-venv",
dest="require_venv",
action="store_true",
default=False,
help=(
"Allow pip to only run in a virtual environment; "
"exit with an error otherwise."
),
help=SUPPRESS_HELP,
)
verbose: Callable[..., Option] = partial(
@@ -964,12 +961,7 @@ use_deprecated_feature: Callable[..., Option] = partial(
metavar="feature",
action="append",
default=[],
choices=[
"legacy-resolver",
"out-of-tree-build",
"backtrack-on-build-failures",
"html5lib",
],
choices=["legacy-resolver", "out-of-tree-build"],
help=("Enable deprecated functionality, that will be removed in the future."),
)
@@ -1,23 +1,10 @@
import functools
import itertools
import sys
from signal import SIGINT, default_int_handler, signal
from typing import Any, Callable, Iterator, Optional, Tuple
from typing import Any
from pip._vendor.progress.bar import Bar, FillingCirclesBar, IncrementalBar
from pip._vendor.progress.spinner import Spinner
from pip._vendor.rich.progress import (
BarColumn,
DownloadColumn,
FileSizeColumn,
Progress,
ProgressColumn,
SpinnerColumn,
TextColumn,
TimeElapsedColumn,
TimeRemainingColumn,
TransferSpeedColumn,
)
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.logging import get_indentation
@@ -30,8 +17,6 @@ try:
except Exception:
colorama = None
DownloadProgressRenderer = Callable[[Iterator[bytes]], Iterator[bytes]]
def _select_progress_class(preferred: Bar, fallback: Bar) -> Bar:
encoding = getattr(preferred.file, "encoding", None)
@@ -258,64 +243,8 @@ BAR_TYPES = {
}
def _legacy_progress_bar(
progress_bar: str, max: Optional[int]
) -> DownloadProgressRenderer:
def DownloadProgressProvider(progress_bar, max=None): # type: ignore
if max is None or max == 0:
return BAR_TYPES[progress_bar][1]().iter # type: ignore
return BAR_TYPES[progress_bar][1]().iter
else:
return BAR_TYPES[progress_bar][0](max=max).iter
#
# Modern replacement, for our legacy progress bars.
#
def _rich_progress_bar(
iterable: Iterator[bytes],
*,
bar_type: str,
size: int,
) -> Iterator[bytes]:
assert bar_type == "on", "This should only be used in the default mode."
if not size:
total = float("inf")
columns: Tuple[ProgressColumn, ...] = (
TextColumn("[progress.description]{task.description}"),
SpinnerColumn("line", speed=1.5),
FileSizeColumn(),
TransferSpeedColumn(),
TimeElapsedColumn(),
)
else:
total = size
columns = (
TextColumn("[progress.description]{task.description}"),
BarColumn(),
DownloadColumn(),
TransferSpeedColumn(),
TextColumn("eta"),
TimeRemainingColumn(),
)
progress = Progress(*columns, refresh_per_second=30)
task_id = progress.add_task(" " * (get_indentation() + 2), total=total)
with progress:
for chunk in iterable:
yield chunk
progress.update(task_id, advance=len(chunk))
def get_download_progress_renderer(
*, bar_type: str, size: Optional[int] = None
) -> DownloadProgressRenderer:
"""Get an object that can be used to render the download progress.
Returns a callable, that takes an iterable to "wrap".
"""
if bar_type == "on":
return functools.partial(_rich_progress_bar, bar_type=bar_type, size=size)
elif bar_type == "off":
return iter # no-op, when passed an iterator
else:
return _legacy_progress_bar(bar_type, size)
@@ -227,31 +227,6 @@ class RequirementCommand(IndexGroupCommand):
return "2020-resolver"
@staticmethod
def determine_build_failure_suppression(options: Values) -> bool:
"""Determines whether build failures should be suppressed and backtracked on."""
if "backtrack-on-build-failures" not in options.deprecated_features_enabled:
return False
if "legacy-resolver" in options.deprecated_features_enabled:
raise CommandError("Cannot backtrack with legacy resolver.")
deprecated(
reason=(
"Backtracking on build failures can mask issues related to how "
"a package generates metadata or builds a wheel. This flag will "
"be removed in pip 22.2."
),
gone_in=None,
replacement=(
"avoiding known-bad versions by explicitly telling pip to ignore them "
"(either directly as requirements, or via a constraints file)"
),
feature_flag=None,
issue=10655,
)
return True
@classmethod
def make_requirement_preparer(
cls,
@@ -262,7 +237,6 @@ class RequirementCommand(IndexGroupCommand):
finder: PackageFinder,
use_user_site: bool,
download_dir: Optional[str] = None,
verbosity: int = 0,
) -> RequirementPreparer:
"""
Create a RequirementPreparer instance for the given parameters.
@@ -302,13 +276,6 @@ class RequirementCommand(IndexGroupCommand):
gone_in="22.1",
)
if options.progress_bar not in {"on", "off"}:
deprecated(
reason="Custom progress bar styles are deprecated",
replacement="to use the default progress bar style.",
gone_in="22.1",
)
return RequirementPreparer(
build_dir=temp_build_dir_path,
src_dir=options.src_dir,
@@ -321,7 +288,6 @@ class RequirementCommand(IndexGroupCommand):
require_hashes=options.require_hashes,
use_user_site=use_user_site,
lazy_wheel=lazy_wheel,
verbosity=verbosity,
in_tree_build=in_tree_build,
)
@@ -348,7 +314,6 @@ class RequirementCommand(IndexGroupCommand):
isolated=options.isolated_mode,
use_pep517=use_pep517,
)
suppress_build_failures = cls.determine_build_failure_suppression(options)
resolver_variant = cls.determine_resolver_variant(options)
# The long import name and duplicated invocation is needed to convince
# Mypy into correctly typechecking. Otherwise it would complain the
@@ -368,7 +333,6 @@ class RequirementCommand(IndexGroupCommand):
force_reinstall=force_reinstall,
upgrade_strategy=upgrade_strategy,
py_version_info=py_version_info,
suppress_build_failures=suppress_build_failures,
)
import pip._internal.resolution.legacy.resolver
@@ -502,5 +466,4 @@ class RequirementCommand(IndexGroupCommand):
link_collector=link_collector,
selection_prefs=selection_prefs,
target_python=target_python,
use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
)
@@ -113,7 +113,6 @@ class DownloadCommand(RequirementCommand):
finder=finder,
download_dir=options.download_dir,
use_user_site=False,
verbosity=self.verbosity,
)
resolver = self.make_resolver(
@@ -97,7 +97,6 @@ class IndexCommand(IndexGroupCommand):
link_collector=link_collector,
selection_prefs=selection_prefs,
target_python=target_python,
use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
)
def get_available_package_versions(self, options: Values, args: List[Any]) -> None:
@@ -319,7 +319,6 @@ class InstallCommand(RequirementCommand):
session=session,
finder=finder,
use_user_site=options.use_user_site,
verbosity=self.verbosity,
)
resolver = self.make_resolver(
preparer=preparer,
@@ -16,6 +16,7 @@ from pip._internal.models.selection_prefs import SelectionPreferences
from pip._internal.network.session import PipSession
from pip._internal.utils.compat import stdlib_pkgs
from pip._internal.utils.misc import tabulate, write_output
from pip._internal.utils.parallel import map_multithread
if TYPE_CHECKING:
from pip._internal.metadata.base import DistributionVersion
@@ -149,7 +150,6 @@ class ListCommand(IndexGroupCommand):
return PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
)
def run(self, options: Values, args: List[str]) -> int:
@@ -254,7 +254,7 @@ class ListCommand(IndexGroupCommand):
dist.latest_filetype = typ
return dist
for dist in map(latest_info, packages):
for dist in map_multithread(latest_info, packages):
if dist is not None:
yield dist
@@ -1,6 +1,8 @@
import csv
import logging
import pathlib
from optparse import Values
from typing import Iterator, List, NamedTuple, Optional
from typing import Iterator, List, NamedTuple, Optional, Tuple
from pip._vendor.packaging.utils import canonicalize_name
@@ -67,6 +69,33 @@ class _PackageInfo(NamedTuple):
files: Optional[List[str]]
def _convert_legacy_entry(entry: Tuple[str, ...], info: Tuple[str, ...]) -> str:
"""Convert a legacy installed-files.txt path into modern RECORD path.
The legacy format stores paths relative to the info directory, while the
modern format stores paths relative to the package root, e.g. the
site-packages directory.
:param entry: Path parts of the installed-files.txt entry.
:param info: Path parts of the egg-info directory relative to package root.
:returns: The converted entry.
For best compatibility with symlinks, this does not use ``abspath()`` or
``Path.resolve()``, but tries to work with path parts:
1. While ``entry`` starts with ``..``, remove the equal amounts of parts
from ``info``; if ``info`` is empty, start appending ``..`` instead.
2. Join the two directly.
"""
while entry and entry[0] == "..":
if not info or info[-1] == "..":
info += ("..",)
else:
info = info[:-1]
entry = entry[1:]
return str(pathlib.Path(*info, *entry))
def search_packages_info(query: List[str]) -> Iterator[_PackageInfo]:
"""
Gather details from installed distributions. Print distribution name,
@@ -92,6 +121,34 @@ def search_packages_info(query: List[str]) -> Iterator[_PackageInfo]:
in {canonicalize_name(d.name) for d in dist.iter_dependencies()}
)
def _files_from_record(dist: BaseDistribution) -> Optional[Iterator[str]]:
try:
text = dist.read_text("RECORD")
except FileNotFoundError:
return None
# This extra Path-str cast normalizes entries.
return (str(pathlib.Path(row[0])) for row in csv.reader(text.splitlines()))
def _files_from_legacy(dist: BaseDistribution) -> Optional[Iterator[str]]:
try:
text = dist.read_text("installed-files.txt")
except FileNotFoundError:
return None
paths = (p for p in text.splitlines(keepends=False) if p)
root = dist.location
info = dist.info_directory
if root is None or info is None:
return paths
try:
info_rel = pathlib.Path(info).relative_to(root)
except ValueError: # info is not relative to root.
return paths
if not info_rel.parts: # info *is* root.
return paths
return (
_convert_legacy_entry(pathlib.Path(p).parts, info_rel.parts) for p in paths
)
for query_name in query_names:
try:
dist = installed[query_name]
@@ -107,7 +164,7 @@ def search_packages_info(query: List[str]) -> Iterator[_PackageInfo]:
except FileNotFoundError:
entry_points = []
files_iter = dist.iter_declared_entries()
files_iter = _files_from_record(dist) or _files_from_legacy(dist)
if files_iter is None:
files: Optional[List[str]] = None
else:
@@ -128,7 +128,6 @@ class WheelCommand(RequirementCommand):
finder=finder,
download_dir=options.wheel_dir,
use_user_site=False,
verbosity=self.verbosity,
)
resolver = self.make_resolver(
@@ -266,13 +266,14 @@ class Configuration:
# Doing this is useful when modifying and saving files, where we don't
# need to construct a parser.
if os.path.exists(fname):
locale_encoding = locale.getpreferredencoding(False)
try:
parser.read(fname, encoding=locale_encoding)
parser.read(fname)
except UnicodeDecodeError:
# See https://github.com/pypa/pip/issues/4963
raise ConfigurationFileCouldNotBeLoaded(
reason=f"contains invalid {locale_encoding} characters",
reason="contains invalid {} characters".format(
locale.getpreferredencoding(False)
),
fname=fname,
)
except configparser.Error as error:
@@ -11,8 +11,10 @@ class InstalledDistribution(AbstractDistribution):
"""
def get_metadata_distribution(self) -> BaseDistribution:
from pip._internal.metadata.pkg_resources import Distribution as _Dist
assert self.req.satisfied_by is not None, "not actually installed"
return self.req.satisfied_by
return _Dist(self.req.satisfied_by)
def prepare_distribution_metadata(
self, finder: PackageFinder, build_isolation: bool
@@ -19,7 +19,9 @@ class SourceDistribution(AbstractDistribution):
"""
def get_metadata_distribution(self) -> BaseDistribution:
return self.req.get_dist()
from pip._internal.metadata.pkg_resources import Distribution as _Dist
return _Dist(self.req.get_dist())
def prepare_distribution_metadata(
self, finder: PackageFinder, build_isolation: bool
@@ -54,7 +56,7 @@ class SourceDistribution(AbstractDistribution):
self.req.build_env = BuildEnvironment()
self.req.build_env.install_requirements(
finder, pyproject_requires, "overlay", kind="build dependencies"
finder, pyproject_requires, "overlay", "Installing build dependencies"
)
conflicting, missing = self.req.build_env.check_requirements(
self.req.requirements_to_check
@@ -106,7 +108,7 @@ class SourceDistribution(AbstractDistribution):
if conflicting:
self._raise_conflicts("the backend dependencies", conflicting)
self.req.build_env.install_requirements(
finder, missing, "normal", kind="backend dependencies"
finder, missing, "normal", "Installing backend dependencies"
)
def _raise_conflicts(
@@ -1,174 +1,23 @@
"""Exceptions used throughout package.
This module MUST NOT try to import from anything within `pip._internal` to
operate. This is expected to be importable from any/all files within the
subpackage and, thus, should not depend on them.
"""
"""Exceptions used throughout package"""
import configparser
import re
from itertools import chain, groupby, repeat
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from pip._vendor.pkg_resources import Distribution
from pip._vendor.requests.models import Request, Response
from pip._vendor.rich.console import Console, ConsoleOptions, RenderResult
from pip._vendor.rich.markup import escape
from pip._vendor.rich.text import Text
if TYPE_CHECKING:
from hashlib import _Hash
from typing import Literal
from pip._internal.metadata import BaseDistribution
from pip._internal.req.req_install import InstallRequirement
#
# Scaffolding
#
def _is_kebab_case(s: str) -> bool:
return re.match(r"^[a-z]+(-[a-z]+)*$", s) is not None
def _prefix_with_indent(
s: Union[Text, str],
console: Console,
*,
prefix: str,
indent: str,
) -> Text:
if isinstance(s, Text):
text = s
else:
text = console.render_str(s)
return console.render_str(prefix, overflow="ignore") + console.render_str(
f"\n{indent}", overflow="ignore"
).join(text.split(allow_blank=True))
class PipError(Exception):
"""The base pip error."""
"""Base pip exception"""
class DiagnosticPipError(PipError):
"""An error, that presents diagnostic information to the user.
This contains a bunch of logic, to enable pretty presentation of our error
messages. Each error gets a unique reference. Each error can also include
additional context, a hint and/or a note -- which are presented with the
main error message in a consistent style.
This is adapted from the error output styling in `sphinx-theme-builder`.
"""
reference: str
def __init__(
self,
*,
kind: 'Literal["error", "warning"]' = "error",
reference: Optional[str] = None,
message: Union[str, Text],
context: Optional[Union[str, Text]],
hint_stmt: Optional[Union[str, Text]],
note_stmt: Optional[Union[str, Text]] = None,
link: Optional[str] = None,
) -> None:
# Ensure a proper reference is provided.
if reference is None:
assert hasattr(self, "reference"), "error reference not provided!"
reference = self.reference
assert _is_kebab_case(reference), "error reference must be kebab-case!"
self.kind = kind
self.reference = reference
self.message = message
self.context = context
self.note_stmt = note_stmt
self.hint_stmt = hint_stmt
self.link = link
super().__init__(f"<{self.__class__.__name__}: {self.reference}>")
def __repr__(self) -> str:
return (
f"<{self.__class__.__name__}("
f"reference={self.reference!r}, "
f"message={self.message!r}, "
f"context={self.context!r}, "
f"note_stmt={self.note_stmt!r}, "
f"hint_stmt={self.hint_stmt!r}"
")>"
)
def __rich_console__(
self,
console: Console,
options: ConsoleOptions,
) -> RenderResult:
colour = "red" if self.kind == "error" else "yellow"
yield f"[{colour} bold]{self.kind}[/]: [bold]{self.reference}[/]"
yield ""
if not options.ascii_only:
# Present the main message, with relevant context indented.
if self.context is not None:
yield _prefix_with_indent(
self.message,
console,
prefix=f"[{colour}]×[/] ",
indent=f"[{colour}]│[/] ",
)
yield _prefix_with_indent(
self.context,
console,
prefix=f"[{colour}]╰─>[/] ",
indent=f"[{colour}] [/] ",
)
else:
yield _prefix_with_indent(
self.message,
console,
prefix="[red]×[/] ",
indent=" ",
)
else:
yield self.message
if self.context is not None:
yield ""
yield self.context
if self.note_stmt is not None or self.hint_stmt is not None:
yield ""
if self.note_stmt is not None:
yield _prefix_with_indent(
self.note_stmt,
console,
prefix="[magenta bold]note[/]: ",
indent=" ",
)
if self.hint_stmt is not None:
yield _prefix_with_indent(
self.hint_stmt,
console,
prefix="[cyan bold]hint[/]: ",
indent=" ",
)
if self.link is not None:
yield ""
yield f"Link: {self.link}"
#
# Actual Errors
#
class ConfigurationError(PipError):
"""General exception in configuration"""
@@ -181,52 +30,18 @@ class UninstallationError(PipError):
"""General exception during uninstallation"""
class MissingPyProjectBuildRequires(DiagnosticPipError):
"""Raised when pyproject.toml has `build-system`, but no `build-system.requires`."""
reference = "missing-pyproject-build-system-requires"
def __init__(self, *, package: str) -> None:
super().__init__(
message=f"Can not process {escape(package)}",
context=Text(
"This package has an invalid pyproject.toml file.\n"
"The [build-system] table is missing the mandatory `requires` key."
),
note_stmt="This is an issue with the package mentioned above, not pip.",
hint_stmt=Text("See PEP 518 for the detailed specification."),
)
class InvalidPyProjectBuildRequires(DiagnosticPipError):
"""Raised when pyproject.toml an invalid `build-system.requires`."""
reference = "invalid-pyproject-build-system-requires"
def __init__(self, *, package: str, reason: str) -> None:
super().__init__(
message=f"Can not process {escape(package)}",
context=Text(
"This package has an invalid `build-system.requires` key in "
f"pyproject.toml.\n{reason}"
),
note_stmt="This is an issue with the package mentioned above, not pip.",
hint_stmt=Text("See PEP 518 for the detailed specification."),
)
class NoneMetadataError(PipError):
"""Raised when accessing a Distribution's "METADATA" or "PKG-INFO".
This signifies an inconsistency, when the Distribution claims to have
the metadata file (if not, raise ``FileNotFoundError`` instead), but is
not actually able to produce its content. This may be due to permission
errors.
"""
Raised when accessing "METADATA" or "PKG-INFO" metadata for a
pip._vendor.pkg_resources.Distribution object and
`dist.has_metadata('METADATA')` returns True but
`dist.get_metadata('METADATA')` returns None (and similarly for
"PKG-INFO").
"""
def __init__(
self,
dist: "BaseDistribution",
dist: Union[Distribution, "BaseDistribution"],
metadata_name: str,
) -> None:
"""
@@ -317,17 +132,6 @@ class UnsupportedWheel(InstallationError):
"""Unsupported wheel."""
class InvalidWheel(InstallationError):
"""Invalid (e.g. corrupt) wheel."""
def __init__(self, location: str, name: str):
self.location = location
self.name = name
def __str__(self) -> str:
return f"Wheel '{self.name}' located at {self.location} is invalid."
class MetadataInconsistent(InstallationError):
"""Built metadata contains inconsistent information.
@@ -352,78 +156,18 @@ class MetadataInconsistent(InstallationError):
return template.format(self.ireq, self.field, self.f_val, self.m_val)
class LegacyInstallFailure(DiagnosticPipError):
"""Error occurred while executing `setup.py install`"""
class InstallationSubprocessError(InstallationError):
"""A subprocess call failed during installation."""
reference = "legacy-install-failure"
def __init__(self, package_details: str) -> None:
super().__init__(
message="Encountered error while trying to install package.",
context=package_details,
hint_stmt="See above for output from the failure.",
note_stmt="This is an issue with the package mentioned above, not pip.",
)
class InstallationSubprocessError(DiagnosticPipError, InstallationError):
"""A subprocess call failed."""
reference = "subprocess-exited-with-error"
def __init__(
self,
*,
command_description: str,
exit_code: int,
output_lines: Optional[List[str]],
) -> None:
if output_lines is None:
output_prompt = Text("See above for output.")
else:
output_prompt = (
Text.from_markup(f"[red][{len(output_lines)} lines of output][/]\n")
+ Text("".join(output_lines))
+ Text.from_markup(R"[red]\[end of output][/]")
)
super().__init__(
message=(
f"[green]{escape(command_description)}[/] did not run successfully.\n"
f"exit code: {exit_code}"
),
context=output_prompt,
hint_stmt=None,
note_stmt=(
"This error originates from a subprocess, and is likely not a "
"problem with pip."
),
)
self.command_description = command_description
self.exit_code = exit_code
def __init__(self, returncode: int, description: str) -> None:
self.returncode = returncode
self.description = description
def __str__(self) -> str:
return f"{self.command_description} exited with {self.exit_code}"
class MetadataGenerationFailed(InstallationSubprocessError, InstallationError):
reference = "metadata-generation-failed"
def __init__(
self,
*,
package_details: str,
) -> None:
super(InstallationSubprocessError, self).__init__(
message="Encountered error while generating package metadata.",
context=escape(package_details),
hint_stmt="See above for details.",
note_stmt="This is an issue with the package mentioned above, not pip.",
)
def __str__(self) -> str:
return "metadata generation failed"
return (
"Command errored out with exit status {}: {} "
"Check the logs for full command output."
).format(self.returncode, self.description)
class HashErrors(InstallationError):
@@ -12,19 +12,15 @@ import re
import urllib.parse
import urllib.request
import xml.etree.ElementTree
from html.parser import HTMLParser
from optparse import Values
from typing import (
TYPE_CHECKING,
Callable,
Dict,
Iterable,
List,
MutableMapping,
NamedTuple,
Optional,
Sequence,
Tuple,
Union,
)
@@ -43,11 +39,6 @@ from pip._internal.vcs import vcs
from .sources import CandidatesFromPage, LinkSource, build_source
if TYPE_CHECKING:
from typing import Protocol
else:
Protocol = object
logger = logging.getLogger(__name__)
HTMLElement = xml.etree.ElementTree.Element
@@ -172,8 +163,6 @@ def _determine_base_url(document: HTMLElement, page_url: str) -> str:
:param document: An HTML document representation. The current
implementation expects the result of ``html5lib.parse()``.
:param page_url: The URL of the HTML document.
TODO: Remove when `html5lib` is dropped.
"""
for base in document.findall(".//base"):
href = base.get("href")
@@ -245,20 +234,20 @@ def _clean_link(url: str) -> str:
def _create_link_from_element(
element_attribs: Dict[str, Optional[str]],
anchor: HTMLElement,
page_url: str,
base_url: str,
) -> Optional[Link]:
"""
Convert an anchor element's attributes in a simple repository page to a Link.
Convert an anchor element in a simple repository page to a Link.
"""
href = element_attribs.get("href")
href = anchor.get("href")
if not href:
return None
url = _clean_link(urllib.parse.urljoin(base_url, href))
pyrequire = element_attribs.get("data-requires-python")
yanked_reason = element_attribs.get("data-yanked")
pyrequire = anchor.get("data-requires-python")
yanked_reason = anchor.get("data-yanked")
link = Link(
url,
@@ -282,14 +271,9 @@ class CacheablePageContent:
return hash(self.page.url)
class ParseLinks(Protocol):
def __call__(
self, page: "HTMLPage", use_deprecated_html5lib: bool
) -> Iterable[Link]:
...
def with_cached_html_pages(fn: ParseLinks) -> ParseLinks:
def with_cached_html_pages(
fn: Callable[["HTMLPage"], Iterable[Link]],
) -> Callable[["HTMLPage"], List[Link]]:
"""
Given a function that parses an Iterable[Link] from an HTMLPage, cache the
function's result (keyed by CacheablePageContent), unless the HTMLPage
@@ -297,25 +281,22 @@ def with_cached_html_pages(fn: ParseLinks) -> ParseLinks:
"""
@functools.lru_cache(maxsize=None)
def wrapper(
cacheable_page: CacheablePageContent, use_deprecated_html5lib: bool
) -> List[Link]:
return list(fn(cacheable_page.page, use_deprecated_html5lib))
def wrapper(cacheable_page: CacheablePageContent) -> List[Link]:
return list(fn(cacheable_page.page))
@functools.wraps(fn)
def wrapper_wrapper(page: "HTMLPage", use_deprecated_html5lib: bool) -> List[Link]:
def wrapper_wrapper(page: "HTMLPage") -> List[Link]:
if page.cache_link_parsing:
return wrapper(CacheablePageContent(page), use_deprecated_html5lib)
return list(fn(page, use_deprecated_html5lib))
return wrapper(CacheablePageContent(page))
return list(fn(page))
return wrapper_wrapper
def _parse_links_html5lib(page: "HTMLPage") -> Iterable[Link]:
@with_cached_html_pages
def parse_links(page: "HTMLPage") -> Iterable[Link]:
"""
Parse an HTML document, and yield its anchor elements as Link objects.
TODO: Remove when `html5lib` is dropped.
"""
document = html5lib.parse(
page.content,
@@ -326,33 +307,6 @@ def _parse_links_html5lib(page: "HTMLPage") -> Iterable[Link]:
url = page.url
base_url = _determine_base_url(document, url)
for anchor in document.findall(".//a"):
link = _create_link_from_element(
anchor.attrib,
page_url=url,
base_url=base_url,
)
if link is None:
continue
yield link
@with_cached_html_pages
def parse_links(page: "HTMLPage", use_deprecated_html5lib: bool) -> Iterable[Link]:
"""
Parse an HTML document, and yield its anchor elements as Link objects.
"""
if use_deprecated_html5lib:
yield from _parse_links_html5lib(page)
return
parser = HTMLLinkParser(page.url)
encoding = page.encoding or "utf-8"
parser.feed(page.content.decode(encoding))
url = page.url
base_url = parser.base_url or url
for anchor in parser.anchors:
link = _create_link_from_element(
anchor,
page_url=url,
@@ -389,34 +343,6 @@ class HTMLPage:
return redact_auth_from_url(self.url)
class HTMLLinkParser(HTMLParser):
"""
HTMLParser that keeps the first base HREF and a list of all anchor
elements' attributes.
"""
def __init__(self, url: str) -> None:
super().__init__(convert_charrefs=True)
self.url: str = url
self.base_url: Optional[str] = None
self.anchors: List[Dict[str, Optional[str]]] = []
def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
if tag == "base" and self.base_url is None:
href = self.get_href(attrs)
if href is not None:
self.base_url = href
elif tag == "a":
self.anchors.append(dict(attrs))
def get_href(self, attrs: List[Tuple[str, Optional[str]]]) -> Optional[str]:
for name, value in attrs:
if name == "href":
return value
return None
def _handle_get_page_fail(
link: Link,
reason: Union[str, Exception],
@@ -37,6 +37,7 @@ from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import build_netloc
from pip._internal.utils.packaging import check_requires_python
from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS
from pip._internal.utils.urls import url_to_path
__all__ = ["FormatControl", "BestCandidateResult", "PackageFinder"]
@@ -580,7 +581,6 @@ class PackageFinder:
link_collector: LinkCollector,
target_python: TargetPython,
allow_yanked: bool,
use_deprecated_html5lib: bool,
format_control: Optional[FormatControl] = None,
candidate_prefs: Optional[CandidatePreferences] = None,
ignore_requires_python: Optional[bool] = None,
@@ -605,7 +605,6 @@ class PackageFinder:
self._ignore_requires_python = ignore_requires_python
self._link_collector = link_collector
self._target_python = target_python
self._use_deprecated_html5lib = use_deprecated_html5lib
self.format_control = format_control
@@ -622,8 +621,6 @@ class PackageFinder:
link_collector: LinkCollector,
selection_prefs: SelectionPreferences,
target_python: Optional[TargetPython] = None,
*,
use_deprecated_html5lib: bool,
) -> "PackageFinder":
"""Create a PackageFinder.
@@ -648,7 +645,6 @@ class PackageFinder:
allow_yanked=selection_prefs.allow_yanked,
format_control=selection_prefs.format_control,
ignore_requires_python=selection_prefs.ignore_requires_python,
use_deprecated_html5lib=use_deprecated_html5lib,
)
@property
@@ -770,7 +766,7 @@ class PackageFinder:
if html_page is None:
return []
page_links = list(parse_links(html_page, self._use_deprecated_html5lib))
page_links = list(parse_links(html_page))
with indent_log():
package_links = self.evaluate_links(
@@ -820,14 +816,7 @@ class PackageFinder:
)
if logger.isEnabledFor(logging.DEBUG) and file_candidates:
paths = []
for candidate in file_candidates:
assert candidate.link.url # we need to have a URL
try:
paths.append(candidate.link.file_path)
except Exception:
paths.append(candidate.link.url) # it's not a local file
paths = [url_to_path(c.link.url) for c in file_candidates]
logger.debug("Local files found: %s", ", ".join(paths))
# This is an intentional priority ordering
@@ -892,7 +881,7 @@ class PackageFinder:
installed_version: Optional[_BaseVersion] = None
if req.satisfied_by is not None:
installed_version = req.satisfied_by.version
installed_version = parse_version(req.satisfied_by.version)
def _format_versions(cand_iter: Iterable[InstallationCandidate]) -> str:
# This repeated parse_version and str() conversion is needed to
@@ -38,34 +38,14 @@ __all__ = [
logger = logging.getLogger(__name__)
if os.environ.get("_PIP_LOCATIONS_NO_WARN_ON_MISMATCH"):
_MISMATCH_LEVEL = logging.DEBUG
else:
_MISMATCH_LEVEL = logging.WARNING
_PLATLIBDIR: str = getattr(sys, "platlibdir", "lib")
_USE_SYSCONFIG_DEFAULT = sys.version_info >= (3, 10)
def _should_use_sysconfig() -> bool:
"""This function determines the value of _USE_SYSCONFIG.
By default, pip uses sysconfig on Python 3.10+.
But Python distributors can override this decision by setting:
sysconfig._PIP_USE_SYSCONFIG = True / False
Rationale in https://github.com/pypa/pip/issues/10647
This is a function for testability, but should be constant during any one
run.
"""
return bool(getattr(sysconfig, "_PIP_USE_SYSCONFIG", _USE_SYSCONFIG_DEFAULT))
_USE_SYSCONFIG = _should_use_sysconfig()
# Be noisy about incompatibilities if this platforms "should" be using
# sysconfig, but is explicitly opting out and using distutils instead.
if _USE_SYSCONFIG_DEFAULT and not _USE_SYSCONFIG:
_MISMATCH_LEVEL = logging.WARNING
else:
_MISMATCH_LEVEL = logging.DEBUG
_USE_SYSCONFIG = sys.version_info >= (3, 10)
def _looks_like_bpo_44860() -> bool:
@@ -84,8 +64,8 @@ def _looks_like_bpo_44860() -> bool:
def _looks_like_red_hat_patched_platlib_purelib(scheme: Dict[str, str]) -> bool:
platlib = scheme["platlib"]
if "/$platlibdir/" in platlib:
platlib = platlib.replace("/$platlibdir/", f"/{_PLATLIBDIR}/")
if "/$platlibdir/" in platlib and hasattr(sys, "platlibdir"):
platlib = platlib.replace("/$platlibdir/", f"/{sys.platlibdir}/")
if "/lib64/" not in platlib:
return False
unpatched = platlib.replace("/lib64/", "/lib/")
@@ -135,22 +115,6 @@ def _looks_like_red_hat_scheme() -> bool:
)
@functools.lru_cache(maxsize=None)
def _looks_like_slackware_scheme() -> bool:
"""Slackware patches sysconfig but fails to patch distutils and site.
Slackware changes sysconfig's user scheme to use ``"lib64"`` for the lib
path, but does not do the same to the site module.
"""
if user_site is None: # User-site not available.
return False
try:
paths = sysconfig.get_paths(scheme="posix_user", expand=False)
except KeyError: # User-site not available.
return False
return "/lib64/" in paths["purelib"] and "/lib64/" not in user_site
@functools.lru_cache(maxsize=None)
def _looks_like_msys2_mingw_scheme() -> bool:
"""MSYS2 patches distutils and sysconfig to use a UNIX-like scheme.
@@ -306,17 +270,6 @@ def get_scheme(
if skip_bpo_44860:
continue
# Slackware incorrectly patches posix_user to use lib64 instead of lib,
# but not usersite to match the location.
skip_slackware_user_scheme = (
user
and k in ("platlib", "purelib")
and not WINDOWS
and _looks_like_slackware_scheme()
)
if skip_slackware_user_scheme:
continue
# Both Debian and Red Hat patch Python to place the system site under
# /usr/local instead of /usr. Debian also places lib in dist-packages
# instead of site-packages, but the /usr/local check should cover it.
@@ -467,13 +420,6 @@ def _deduplicated(v1: str, v2: str) -> List[str]:
return [v1, v2]
def _looks_like_apple_library(path: str) -> bool:
"""Apple patches sysconfig to *always* look under */Library/Python*."""
if sys.platform[:6] != "darwin":
return False
return path == f"/Library/Python/{get_major_minor_version()}/site-packages"
def get_prefixed_libs(prefix: str) -> List[str]:
"""Return the lib locations under ``prefix``."""
new_pure, new_plat = _sysconfig.get_prefixed_libs(prefix)
@@ -481,26 +427,6 @@ def get_prefixed_libs(prefix: str) -> List[str]:
return _deduplicated(new_pure, new_plat)
old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
old_lib_paths = _deduplicated(old_pure, old_plat)
# Apple's Python (shipped with Xcode and Command Line Tools) hard-code
# platlib and purelib to '/Library/Python/X.Y/site-packages'. This will
# cause serious build isolation bugs when Apple starts shipping 3.10 because
# pip will install build backends to the wrong location. This tells users
# who is at fault so Apple may notice it and fix the issue in time.
if all(_looks_like_apple_library(p) for p in old_lib_paths):
deprecated(
reason=(
"Python distributed by Apple's Command Line Tools incorrectly "
"patches sysconfig to always point to '/Library/Python'. This "
"will cause build isolation to operate incorrectly on Python "
"3.10 or later. Please help report this to Apple so they can "
"fix this. https://developer.apple.com/bug-reporting/"
),
replacement=None,
gone_in=None,
)
return old_lib_paths
warned = [
_warn_if_mismatch(
@@ -517,4 +443,4 @@ def get_prefixed_libs(prefix: str) -> List[str]:
if any(warned):
_log_context(prefix=prefix)
return old_lib_paths
return _deduplicated(old_pure, old_plat)
@@ -38,17 +38,6 @@ def get_environment(paths: Optional[List[str]]) -> BaseEnvironment:
return Environment.from_paths(paths)
def get_directory_distribution(directory: str) -> BaseDistribution:
"""Get the distribution metadata representation in the specified directory.
This returns a Distribution instance from the chosen backend based on
the given on-disk ``.dist-info`` directory.
"""
from .pkg_resources import Distribution
return Distribution.from_directory(directory)
def get_wheel_distribution(wheel: Wheel, canonical_name: str) -> BaseDistribution:
"""Get the representation of the specified wheel's distribution metadata.
@@ -1,8 +1,6 @@
import csv
import email.message
import json
import logging
import pathlib
import re
import zipfile
from typing import (
@@ -14,7 +12,6 @@ from typing import (
Iterator,
List,
Optional,
Tuple,
Union,
)
@@ -23,19 +20,13 @@ from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
from pip._vendor.packaging.utils import NormalizedName
from pip._vendor.packaging.version import LegacyVersion, Version
from pip._internal.exceptions import NoneMetadataError
from pip._internal.locations import site_packages, user_site
from pip._internal.models.direct_url import (
DIRECT_URL_METADATA_NAME,
DirectUrl,
DirectUrlValidationError,
)
from pip._internal.utils.compat import stdlib_pkgs # TODO: Move definition here.
from pip._internal.utils.egg_link import (
egg_link_path_from_location,
egg_link_path_from_sys_path,
)
from pip._internal.utils.misc import is_local, normalize_path
from pip._internal.utils.egg_link import egg_link_path_from_sys_path
from pip._internal.utils.urls import url_to_path
if TYPE_CHECKING:
@@ -45,8 +36,6 @@ else:
DistributionVersion = Union[LegacyVersion, Version]
InfoPath = Union[str, pathlib.PurePosixPath]
logger = logging.getLogger(__name__)
@@ -64,36 +53,6 @@ class BaseEntryPoint(Protocol):
raise NotImplementedError()
def _convert_installed_files_path(
entry: Tuple[str, ...],
info: Tuple[str, ...],
) -> str:
"""Convert a legacy installed-files.txt path into modern RECORD path.
The legacy format stores paths relative to the info directory, while the
modern format stores paths relative to the package root, e.g. the
site-packages directory.
:param entry: Path parts of the installed-files.txt entry.
:param info: Path parts of the egg-info directory relative to package root.
:returns: The converted entry.
For best compatibility with symlinks, this does not use ``abspath()`` or
``Path.resolve()``, but tries to work with path parts:
1. While ``entry`` starts with ``..``, remove the equal amounts of parts
from ``info``; if ``info`` is empty, start appending ``..`` instead.
2. Join the two directly.
"""
while entry and entry[0] == "..":
if not info or info[-1] == "..":
info += ("..",)
else:
info = info[:-1]
entry = entry[1:]
return str(pathlib.Path(*info, *entry))
class BaseDistribution(Protocol):
def __repr__(self) -> str:
return f"{self.raw_name} {self.version} ({self.location})"
@@ -138,28 +97,8 @@ class BaseDistribution(Protocol):
return None
@property
def installed_location(self) -> Optional[str]:
"""The distribution's "installed" location.
This should generally be a ``site-packages`` directory. This is
usually ``dist.location``, except for legacy develop-installed packages,
where ``dist.location`` is the source code location, and this is where
the ``.egg-link`` file is.
The returned location is normalized (in particular, with symlinks removed).
"""
egg_link = egg_link_path_from_location(self.raw_name)
if egg_link:
location = egg_link
elif self.location:
location = self.location
else:
return None
return normalize_path(location)
@property
def info_location(self) -> Optional[str]:
"""Location of the .[egg|dist]-info directory or file.
def info_directory(self) -> Optional[str]:
"""Location of the .[egg|dist]-info directory.
Similarly to ``location``, a string value is not necessarily a
filesystem path. ``None`` means the distribution is created in-memory.
@@ -173,65 +112,6 @@ class BaseDistribution(Protocol):
"""
raise NotImplementedError()
@property
def installed_by_distutils(self) -> bool:
"""Whether this distribution is installed with legacy distutils format.
A distribution installed with "raw" distutils not patched by setuptools
uses one single file at ``info_location`` to store metadata. We need to
treat this specially on uninstallation.
"""
info_location = self.info_location
if not info_location:
return False
return pathlib.Path(info_location).is_file()
@property
def installed_as_egg(self) -> bool:
"""Whether this distribution is installed as an egg.
This usually indicates the distribution was installed by (older versions
of) easy_install.
"""
location = self.location
if not location:
return False
return location.endswith(".egg")
@property
def installed_with_setuptools_egg_info(self) -> bool:
"""Whether this distribution is installed with the ``.egg-info`` format.
This usually indicates the distribution was installed with setuptools
with an old pip version or with ``single-version-externally-managed``.
Note that this ensure the metadata store is a directory. distutils can
also installs an ``.egg-info``, but as a file, not a directory. This
property is *False* for that case. Also see ``installed_by_distutils``.
"""
info_location = self.info_location
if not info_location:
return False
if not info_location.endswith(".egg-info"):
return False
return pathlib.Path(info_location).is_dir()
@property
def installed_with_dist_info(self) -> bool:
"""Whether this distribution is installed with the "modern format".
This indicates a "modern" installation, e.g. storing metadata in the
``.dist-info`` directory. This applies to installations made by
setuptools (but through pip, not directly), or anything using the
standardized build backend interface (PEP 517).
"""
info_location = self.info_location
if not info_location:
return False
if not info_location.endswith(".dist-info"):
return False
return pathlib.Path(info_location).is_dir()
@property
def canonical_name(self) -> NormalizedName:
raise NotImplementedError()
@@ -240,14 +120,6 @@ class BaseDistribution(Protocol):
def version(self) -> DistributionVersion:
raise NotImplementedError()
@property
def setuptools_filename(self) -> str:
"""Convert a project name to its setuptools-compatible filename.
This is a copy of ``pkg_resources.to_filename()`` for compatibility.
"""
return self.raw_name.replace("-", "_")
@property
def direct_url(self) -> Optional[DirectUrl]:
"""Obtain a DirectUrl from this distribution.
@@ -276,15 +148,7 @@ class BaseDistribution(Protocol):
@property
def installer(self) -> str:
try:
installer_text = self.read_text("INSTALLER")
except (OSError, ValueError, NoneMetadataError):
return "" # Fail silently if the installer file cannot be read.
for line in installer_text.splitlines():
cleaned_line = line.strip()
if cleaned_line:
return cleaned_line
return ""
raise NotImplementedError()
@property
def editable(self) -> bool:
@@ -292,46 +156,21 @@ class BaseDistribution(Protocol):
@property
def local(self) -> bool:
"""If distribution is installed in the current virtual environment.
Always True if we're not in a virtualenv.
"""
if self.installed_location is None:
return False
return is_local(self.installed_location)
raise NotImplementedError()
@property
def in_usersite(self) -> bool:
if self.installed_location is None or user_site is None:
return False
return self.installed_location.startswith(normalize_path(user_site))
raise NotImplementedError()
@property
def in_site_packages(self) -> bool:
if self.installed_location is None or site_packages is None:
return False
return self.installed_location.startswith(normalize_path(site_packages))
def is_file(self, path: InfoPath) -> bool:
"""Check whether an entry in the info directory is a file."""
raise NotImplementedError()
def iterdir(self, path: InfoPath) -> Iterator[pathlib.PurePosixPath]:
"""Iterate through a directory in the info directory.
def read_text(self, name: str) -> str:
"""Read a file in the .dist-info (or .egg-info) directory.
Each item yielded would be a path relative to the info directory.
:raise FileNotFoundError: If ``name`` does not exist in the directory.
:raise NotADirectoryError: If ``name`` does not point to a directory.
"""
raise NotImplementedError()
def read_text(self, path: InfoPath) -> str:
"""Read a file in the info directory.
:raise FileNotFoundError: If ``name`` does not exist in the directory.
:raise NoneMetadataError: If ``name`` exists in the info directory, but
cannot be read.
Should raise ``FileNotFoundError`` if ``name`` does not exist in the
metadata directory.
"""
raise NotImplementedError()
@@ -340,13 +179,7 @@ class BaseDistribution(Protocol):
@property
def metadata(self) -> email.message.Message:
"""Metadata of distribution parsed from e.g. METADATA or PKG-INFO.
This should return an empty message if the metadata file is unavailable.
:raises NoneMetadataError: If the metadata file is available, but does
not contain valid metadata.
"""
"""Metadata of distribution parsed from e.g. METADATA or PKG-INFO."""
raise NotImplementedError()
@property
@@ -396,51 +229,6 @@ class BaseDistribution(Protocol):
"""
raise NotImplementedError()
def _iter_declared_entries_from_record(self) -> Optional[Iterator[str]]:
try:
text = self.read_text("RECORD")
except FileNotFoundError:
return None
# This extra Path-str cast normalizes entries.
return (str(pathlib.Path(row[0])) for row in csv.reader(text.splitlines()))
def _iter_declared_entries_from_legacy(self) -> Optional[Iterator[str]]:
try:
text = self.read_text("installed-files.txt")
except FileNotFoundError:
return None
paths = (p for p in text.splitlines(keepends=False) if p)
root = self.location
info = self.info_location
if root is None or info is None:
return paths
try:
info_rel = pathlib.Path(info).relative_to(root)
except ValueError: # info is not relative to root.
return paths
if not info_rel.parts: # info *is* root.
return paths
return (
_convert_installed_files_path(pathlib.Path(p).parts, info_rel.parts)
for p in paths
)
def iter_declared_entries(self) -> Optional[Iterator[str]]:
"""Iterate through file entires declared in this distribution.
For modern .dist-info distributions, this is the files listed in the
``RECORD`` metadata file. For legacy setuptools distributions, this
comes from ``installed-files.txt``, with entries normalized to be
compatible with the format used by ``RECORD``.
:return: An iterator for listed entries, or None if the distribution
contains neither ``RECORD`` nor ``installed-files.txt``.
"""
return (
self._iter_declared_entries_from_record()
or self._iter_declared_entries_from_legacy()
)
class BaseEnvironment:
"""An environment containing distributions to introspect."""
@@ -454,11 +242,7 @@ class BaseEnvironment:
raise NotImplementedError()
def get_distribution(self, name: str) -> Optional["BaseDistribution"]:
"""Given a requirement name, return the installed distributions.
The name may not be normalized. The implementation must canonicalize
it for lookup.
"""
"""Given a requirement name, return the installed distributions."""
raise NotImplementedError()
def _iter_distributions(self) -> Iterator["BaseDistribution"]:
@@ -1,26 +1,21 @@
import email.message
import email.parser
import logging
import os
import pathlib
import zipfile
from typing import Collection, Iterable, Iterator, List, Mapping, NamedTuple, Optional
from typing import Collection, Iterable, Iterator, List, NamedTuple, Optional
from pip._vendor import pkg_resources
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import parse as parse_version
from pip._internal.exceptions import InvalidWheel, NoneMetadataError, UnsupportedWheel
from pip._internal.utils.misc import display_path
from pip._internal.utils.wheel import parse_wheel, read_wheel_metadata_file
from pip._internal.utils import misc # TODO: Move definition here.
from pip._internal.utils.packaging import get_installer, get_metadata
from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
from .base import (
BaseDistribution,
BaseEntryPoint,
BaseEnvironment,
DistributionVersion,
InfoPath,
Wheel,
)
@@ -33,91 +28,14 @@ class EntryPoint(NamedTuple):
group: str
class WheelMetadata:
"""IMetadataProvider that reads metadata files from a dictionary.
This also maps metadata decoding exceptions to our internal exception type.
"""
def __init__(self, metadata: Mapping[str, bytes], wheel_name: str) -> None:
self._metadata = metadata
self._wheel_name = wheel_name
def has_metadata(self, name: str) -> bool:
return name in self._metadata
def get_metadata(self, name: str) -> str:
try:
return self._metadata[name].decode()
except UnicodeDecodeError as e:
# Augment the default error with the origin of the file.
raise UnsupportedWheel(
f"Error decoding metadata for {self._wheel_name}: {e} in {name} file"
)
def get_metadata_lines(self, name: str) -> Iterable[str]:
return pkg_resources.yield_lines(self.get_metadata(name))
def metadata_isdir(self, name: str) -> bool:
return False
def metadata_listdir(self, name: str) -> List[str]:
return []
def run_script(self, script_name: str, namespace: str) -> None:
pass
class Distribution(BaseDistribution):
def __init__(self, dist: pkg_resources.Distribution) -> None:
self._dist = dist
@classmethod
def from_directory(cls, directory: str) -> "Distribution":
dist_dir = directory.rstrip(os.sep)
# Build a PathMetadata object, from path to metadata. :wink:
base_dir, dist_dir_name = os.path.split(dist_dir)
metadata = pkg_resources.PathMetadata(base_dir, dist_dir)
# Determine the correct Distribution object type.
if dist_dir.endswith(".egg-info"):
dist_cls = pkg_resources.Distribution
dist_name = os.path.splitext(dist_dir_name)[0]
else:
assert dist_dir.endswith(".dist-info")
dist_cls = pkg_resources.DistInfoDistribution
dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
dist = dist_cls(base_dir, project_name=dist_name, metadata=metadata)
return cls(dist)
@classmethod
def from_wheel(cls, wheel: Wheel, name: str) -> "Distribution":
"""Load the distribution from a given wheel.
:raises InvalidWheel: Whenever loading of the wheel causes a
:py:exc:`zipfile.BadZipFile` exception to be thrown.
:raises UnsupportedWheel: If the wheel is a valid zip, but malformed
internally.
"""
try:
with wheel.as_zipfile() as zf:
info_dir, _ = parse_wheel(zf, name)
metadata_text = {
path.split("/", 1)[-1]: read_wheel_metadata_file(zf, path)
for path in zf.namelist()
if path.startswith(f"{info_dir}/")
}
except zipfile.BadZipFile as e:
raise InvalidWheel(wheel.location, name) from e
except UnsupportedWheel as e:
raise UnsupportedWheel(f"{name} has an invalid wheel, {e}")
dist = pkg_resources.DistInfoDistribution(
location=wheel.location,
metadata=WheelMetadata(metadata_text, wheel.location),
project_name=name,
)
with wheel.as_zipfile() as zf:
dist = pkg_resources_distribution_for_wheel(zf, name, wheel.location)
return cls(dist)
@property
@@ -125,19 +43,9 @@ class Distribution(BaseDistribution):
return self._dist.location
@property
def info_location(self) -> Optional[str]:
def info_directory(self) -> Optional[str]:
return self._dist.egg_info
@property
def installed_by_distutils(self) -> bool:
# A distutils-installed distribution is provided by FileMetadata. This
# provider has a "path" attribute not present anywhere else. Not the
# best introspection logic, but pip has been doing this for a long time.
try:
return bool(self._dist._provider.path)
except AttributeError:
return False
@property
def canonical_name(self) -> NormalizedName:
return canonicalize_name(self._dist.project_name)
@@ -146,26 +54,26 @@ class Distribution(BaseDistribution):
def version(self) -> DistributionVersion:
return parse_version(self._dist.version)
def is_file(self, path: InfoPath) -> bool:
return self._dist.has_metadata(str(path))
@property
def installer(self) -> str:
return get_installer(self._dist)
def iterdir(self, path: InfoPath) -> Iterator[pathlib.PurePosixPath]:
name = str(path)
@property
def local(self) -> bool:
return misc.dist_is_local(self._dist)
@property
def in_usersite(self) -> bool:
return misc.dist_in_usersite(self._dist)
@property
def in_site_packages(self) -> bool:
return misc.dist_in_site_packages(self._dist)
def read_text(self, name: str) -> str:
if not self._dist.has_metadata(name):
raise FileNotFoundError(name)
if not self._dist.isdir(name):
raise NotADirectoryError(name)
for child in self._dist.metadata_listdir(name):
yield pathlib.PurePosixPath(path, child)
def read_text(self, path: InfoPath) -> str:
name = str(path)
if not self._dist.has_metadata(name):
raise FileNotFoundError(name)
content = self._dist.get_metadata(name)
if content is None:
raise NoneMetadataError(self, name)
return content
return self._dist.get_metadata(name)
def iter_entry_points(self) -> Iterable[BaseEntryPoint]:
for group, entries in self._dist.get_entry_map().items():
@@ -175,26 +83,7 @@ class Distribution(BaseDistribution):
@property
def metadata(self) -> email.message.Message:
"""
:raises NoneMetadataError: if the distribution reports `has_metadata()`
True but `get_metadata()` returns None.
"""
if isinstance(self._dist, pkg_resources.DistInfoDistribution):
metadata_name = "METADATA"
else:
metadata_name = "PKG-INFO"
try:
metadata = self.read_text(metadata_name)
except FileNotFoundError:
if self.location:
displaying_path = display_path(self.location)
else:
displaying_path = repr(self.location)
logger.warning("No metadata found in %s", displaying_path)
metadata = ""
feed_parser = email.parser.FeedParser()
feed_parser.feed(metadata)
return feed_parser.close()
return get_metadata(self._dist)
def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
if extras: # pkg_resources raises on invalid extras, so we sanitize.
@@ -230,6 +119,7 @@ class Environment(BaseEnvironment):
return None
def get_distribution(self, name: str) -> Optional[BaseDistribution]:
# Search the distribution by looking through the working set.
dist = self._search_distribution(name)
if dist:
@@ -53,7 +53,7 @@ class SafeFileCache(BaseCache):
with open(path, "rb") as f:
return f.read()
def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None:
def set(self, key: str, value: bytes) -> None:
path = self._get_cache_path(key)
with suppressed_cache_errors():
ensure_dir(os.path.dirname(path))
@@ -8,7 +8,7 @@ from typing import Iterable, Optional, Tuple
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
from pip._internal.cli.progress_bars import get_download_progress_renderer
from pip._internal.cli.progress_bars import DownloadProgressProvider
from pip._internal.exceptions import NetworkConnectionError
from pip._internal.models.index import PyPI
from pip._internal.models.link import Link
@@ -65,8 +65,7 @@ def _prepare_download(
if not show_progress:
return chunks
renderer = get_download_progress_renderer(bar_type=progress_bar, size=total_length)
return renderer(chunks)
return DownloadProgressProvider(progress_bar, max=total_length)(chunks)
def sanitize_content_filename(filename: str) -> str:
@@ -6,17 +6,11 @@ import os
from pip._vendor.pep517.wrappers import Pep517HookCaller
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import (
InstallationSubprocessError,
MetadataGenerationFailed,
)
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory
def generate_metadata(
build_env: BuildEnvironment, backend: Pep517HookCaller, details: str
) -> str:
def generate_metadata(build_env: BuildEnvironment, backend: Pep517HookCaller) -> str:
"""Generate metadata using mechanisms described in PEP 517.
Returns the generated metadata directory.
@@ -31,9 +25,6 @@ def generate_metadata(
# consider the possibility that this hook doesn't exist.
runner = runner_with_spinner_message("Preparing metadata (pyproject.toml)")
with backend.subprocess_runner(runner):
try:
distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
except InstallationSubprocessError as error:
raise MetadataGenerationFailed(package_details=details) from error
distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
return os.path.join(metadata_dir, distinfo_dir)
@@ -6,16 +6,12 @@ import os
from pip._vendor.pep517.wrappers import Pep517HookCaller
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import (
InstallationSubprocessError,
MetadataGenerationFailed,
)
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory
def generate_editable_metadata(
build_env: BuildEnvironment, backend: Pep517HookCaller, details: str
build_env: BuildEnvironment, backend: Pep517HookCaller
) -> str:
"""Generate metadata using mechanisms described in PEP 660.
@@ -33,9 +29,6 @@ def generate_editable_metadata(
"Preparing editable metadata (pyproject.toml)"
)
with backend.subprocess_runner(runner):
try:
distinfo_dir = backend.prepare_metadata_for_build_editable(metadata_dir)
except InstallationSubprocessError as error:
raise MetadataGenerationFailed(package_details=details) from error
distinfo_dir = backend.prepare_metadata_for_build_editable(metadata_dir)
return os.path.join(metadata_dir, distinfo_dir)
@@ -6,11 +6,7 @@ import os
from pip._internal.build_env import BuildEnvironment
from pip._internal.cli.spinners import open_spinner
from pip._internal.exceptions import (
InstallationError,
InstallationSubprocessError,
MetadataGenerationFailed,
)
from pip._internal.exceptions import InstallationError
from pip._internal.utils.setuptools_build import make_setuptools_egg_info_args
from pip._internal.utils.subprocess import call_subprocess
from pip._internal.utils.temp_dir import TempDirectory
@@ -60,15 +56,12 @@ def generate_metadata(
with build_env:
with open_spinner("Preparing metadata (setup.py)") as spinner:
try:
call_subprocess(
args,
cwd=source_dir,
command_desc="python setup.py egg_info",
spinner=spinner,
)
except InstallationSubprocessError as error:
raise MetadataGenerationFailed(package_details=details) from error
call_subprocess(
args,
cwd=source_dir,
command_desc="python setup.py egg_info",
spinner=spinner,
)
# Return the .egg-info directory.
return _find_egg_info(egg_info_dir)
@@ -4,7 +4,11 @@ from typing import List, Optional
from pip._internal.cli.spinners import open_spinner
from pip._internal.utils.setuptools_build import make_setuptools_bdist_wheel_args
from pip._internal.utils.subprocess import call_subprocess, format_command_args
from pip._internal.utils.subprocess import (
LOG_DIVIDER,
call_subprocess,
format_command_args,
)
logger = logging.getLogger(__name__)
@@ -24,7 +28,7 @@ def format_command_result(
else:
if not command_output.endswith("\n"):
command_output += "\n"
text += f"Command output:\n{command_output}"
text += f"Command output:\n{command_output}{LOG_DIVIDER}"
return text
@@ -82,7 +86,6 @@ def build_wheel_legacy(
try:
output = call_subprocess(
wheel_args,
command_desc="python setup.py bdist_wheel",
cwd=source_dir,
spinner=spinner,
)
@@ -42,6 +42,5 @@ def install_editable(
with build_env:
call_subprocess(
args,
command_desc="python setup.py develop",
cwd=unpacked_source_directory,
)
@@ -7,8 +7,9 @@ from distutils.util import change_root
from typing import List, Optional, Sequence
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import InstallationError, LegacyInstallFailure
from pip._internal.exceptions import InstallationError
from pip._internal.models.scheme import Scheme
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import ensure_dir
from pip._internal.utils.setuptools_build import make_setuptools_install_args
from pip._internal.utils.subprocess import runner_with_spinner_message
@@ -17,6 +18,10 @@ from pip._internal.utils.temp_dir import TempDirectory
logger = logging.getLogger(__name__)
class LegacyInstallFailure(Exception):
pass
def write_installed_files_from_setuptools_record(
record_lines: List[str],
root: Optional[str],
@@ -93,7 +98,7 @@ def install(
runner = runner_with_spinner_message(
f"Running setup.py install for {req_name}"
)
with build_env:
with indent_log(), build_env:
runner(
cmd=install_args,
cwd=unpacked_source_directory,
@@ -106,7 +111,7 @@ def install(
except Exception as e:
# Signal to the caller that we didn't install the new package
raise LegacyInstallFailure(package_details=req_name) from e
raise LegacyInstallFailure from e
# At this point, we have successfully installed the requirement.
@@ -59,10 +59,10 @@ def _get_prepared_distribution(
return abstract_dist.get_metadata_distribution()
def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
def unpack_vcs_link(link: Link, location: str) -> None:
vcs_backend = vcs.get_backend_for_scheme(link.scheme)
assert vcs_backend is not None
vcs_backend.unpack(location, url=hide_url(link.url), verbosity=verbosity)
vcs_backend.unpack(location, url=hide_url(link.url))
class File:
@@ -175,7 +175,6 @@ def unpack_url(
link: Link,
location: str,
download: Downloader,
verbosity: int,
download_dir: Optional[str] = None,
hashes: Optional[Hashes] = None,
) -> Optional[File]:
@@ -188,7 +187,7 @@ def unpack_url(
"""
# non-editable vcs urls
if link.is_vcs:
unpack_vcs_link(link, location, verbosity=verbosity)
unpack_vcs_link(link, location)
return None
# Once out-of-tree-builds are no longer supported, could potentially
@@ -268,7 +267,6 @@ class RequirementPreparer:
require_hashes: bool,
use_user_site: bool,
lazy_wheel: bool,
verbosity: int,
in_tree_build: bool,
) -> None:
super().__init__()
@@ -297,9 +295,6 @@ class RequirementPreparer:
# Should wheels be downloaded lazily?
self.use_lazy_wheel = lazy_wheel
# How verbose should underlying tooling be?
self.verbosity = verbosity
# Should in-tree builds be used for local paths?
self.in_tree_build = in_tree_build
@@ -530,12 +525,7 @@ class RequirementPreparer:
elif link.url not in self._downloaded:
try:
local_file = unpack_url(
link,
req.source_dir,
self._download,
self.verbosity,
self.download_dir,
hashes,
link, req.source_dir, self._download, self.download_dir, hashes
)
except NetworkConnectionError as exc:
raise InstallationError(
@@ -5,11 +5,7 @@ from typing import Any, List, Optional
from pip._vendor import tomli
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._internal.exceptions import (
InstallationError,
InvalidPyProjectBuildRequires,
MissingPyProjectBuildRequires,
)
from pip._internal.exceptions import InstallationError
def _is_list_of_str(obj: Any) -> bool:
@@ -60,7 +56,7 @@ def load_pyproject_toml(
if has_pyproject:
with open(pyproject_toml, encoding="utf-8") as f:
pp_toml = tomli.loads(f.read())
pp_toml = tomli.load(f)
build_system = pp_toml.get("build-system")
else:
build_system = None
@@ -123,28 +119,47 @@ def load_pyproject_toml(
# Ensure that the build-system section in pyproject.toml conforms
# to PEP 518.
error_template = (
"{package} has a pyproject.toml file that does not comply "
"with PEP 518: {reason}"
)
# Specifying the build-system table but not the requires key is invalid
if "requires" not in build_system:
raise MissingPyProjectBuildRequires(package=req_name)
raise InstallationError(
error_template.format(
package=req_name,
reason=(
"it has a 'build-system' table but not "
"'build-system.requires' which is mandatory in the table"
),
)
)
# Error out if requires is not a list of strings
requires = build_system["requires"]
if not _is_list_of_str(requires):
raise InvalidPyProjectBuildRequires(
package=req_name,
reason="It is not a list of strings.",
raise InstallationError(
error_template.format(
package=req_name,
reason="'build-system.requires' is not a list of strings.",
)
)
# Each requirement must be valid as per PEP 508
for requirement in requires:
try:
Requirement(requirement)
except InvalidRequirement as error:
raise InvalidPyProjectBuildRequires(
package=req_name,
reason=f"It contains an invalid requirement: {requirement!r}",
) from error
except InvalidRequirement:
raise InstallationError(
error_template.format(
package=req_name,
reason=(
"'build-system.requires' contains an invalid "
"requirement: {!r}".format(requirement)
),
)
)
backend = build_system.get("build-backend")
backend_path = build_system.get("backend-path", [])
@@ -16,6 +16,7 @@ from typing import Any, Dict, Optional, Set, Tuple, Union
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.specifiers import Specifier
from pip._vendor.pkg_resources import RequirementParseError, parse_requirements
from pip._internal.exceptions import InstallationError
from pip._internal.models.index import PyPI, TestPyPI
@@ -112,56 +113,31 @@ def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
return package_name, url, set()
def check_first_requirement_in_file(filename: str) -> None:
"""Check if file is parsable as a requirements file.
This is heavily based on ``pkg_resources.parse_requirements``, but
simplified to just check the first meaningful line.
:raises InvalidRequirement: If the first meaningful line cannot be parsed
as an requirement.
"""
with open(filename, encoding="utf-8", errors="ignore") as f:
# Create a steppable iterator, so we can handle \-continuations.
lines = (
line
for line in (line.strip() for line in f)
if line and not line.startswith("#") # Skip blank lines/comments.
)
for line in lines:
# Drop comments -- a hash without a space may be in a URL.
if " #" in line:
line = line[: line.find(" #")]
# If there is a line continuation, drop it, and append the next line.
if line.endswith("\\"):
line = line[:-2].strip() + next(lines, "")
Requirement(line)
return
def deduce_helpful_msg(req: str) -> str:
"""Returns helpful msg in case requirements file does not exist,
or cannot be parsed.
:params req: Requirements file path
"""
if not os.path.exists(req):
return f" File '{req}' does not exist."
msg = " The path does exist. "
# Try to parse and check if it is a requirements file.
try:
check_first_requirement_in_file(req)
except InvalidRequirement:
logger.debug("Cannot parse '%s' as requirements file", req)
msg = ""
if os.path.exists(req):
msg = " The path does exist. "
# Try to parse and check if it is a requirements file.
try:
with open(req) as fp:
# parse first line only
next(parse_requirements(fp.read()))
msg += (
"The argument you provided "
"({}) appears to be a"
" requirements file. If that is the"
" case, use the '-r' flag to install"
" the packages specified within it."
).format(req)
except RequirementParseError:
logger.debug("Cannot parse '%s' as requirements file", req, exc_info=True)
else:
msg += (
f"The argument you provided "
f"({req}) appears to be a"
f" requirements file. If that is the"
f" case, use the '-r' flag to install"
f" the packages specified within it."
)
msg += f" File '{req}' does not exist."
return msg
@@ -10,6 +10,7 @@ import uuid
import zipfile
from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, Union
from pip._vendor import pkg_resources
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.specifiers import SpecifierSet
@@ -17,15 +18,11 @@ from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
from pip._vendor.pep517.wrappers import Pep517HookCaller
from pip._vendor.pkg_resources import Distribution
from pip._internal.build_env import BuildEnvironment, NoOpBuildEnvironment
from pip._internal.exceptions import InstallationError, LegacyInstallFailure
from pip._internal.exceptions import InstallationError
from pip._internal.locations import get_scheme
from pip._internal.metadata import (
BaseDistribution,
get_default_environment,
get_directory_distribution,
)
from pip._internal.models.link import Link
from pip._internal.operations.build.metadata import generate_metadata
from pip._internal.operations.build.metadata_editable import generate_editable_metadata
@@ -35,6 +32,7 @@ from pip._internal.operations.build.metadata_legacy import (
from pip._internal.operations.install.editable_legacy import (
install_editable as install_editable_legacy,
)
from pip._internal.operations.install.legacy import LegacyInstallFailure
from pip._internal.operations.install.legacy import install as install_legacy
from pip._internal.operations.install.wheel import install_wheel
from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path
@@ -49,10 +47,13 @@ from pip._internal.utils.misc import (
ask_path_exists,
backup_dir,
display_path,
dist_in_site_packages,
dist_in_usersite,
get_distribution,
hide_url,
redact_auth_from_url,
)
from pip._internal.utils.packaging import safe_extra
from pip._internal.utils.packaging import get_metadata
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
from pip._internal.utils.virtualenv import running_under_virtualenv
@@ -61,6 +62,32 @@ from pip._internal.vcs import vcs
logger = logging.getLogger(__name__)
def _get_dist(metadata_directory: str) -> Distribution:
"""Return a pkg_resources.Distribution for the provided
metadata directory.
"""
dist_dir = metadata_directory.rstrip(os.sep)
# Build a PathMetadata object, from path to metadata. :wink:
base_dir, dist_dir_name = os.path.split(dist_dir)
metadata = pkg_resources.PathMetadata(base_dir, dist_dir)
# Determine the correct Distribution object type.
if dist_dir.endswith(".egg-info"):
dist_cls = pkg_resources.Distribution
dist_name = os.path.splitext(dist_dir_name)[0]
else:
assert dist_dir.endswith(".dist-info")
dist_cls = pkg_resources.DistInfoDistribution
dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
return dist_cls(
base_dir,
project_name=dist_name,
metadata=metadata,
)
class InstallRequirement:
"""
Represents something that may be installed later on, may have information
@@ -118,15 +145,16 @@ class InstallRequirement:
if extras:
self.extras = extras
elif req:
self.extras = {safe_extra(extra) for extra in req.extras}
self.extras = {pkg_resources.safe_extra(extra) for extra in req.extras}
else:
self.extras = set()
if markers is None and req:
markers = req.marker
self.markers = markers
# This holds the Distribution object if this requirement is already installed.
self.satisfied_by: Optional[BaseDistribution] = None
# This holds the pkg_resources.Distribution object if this requirement
# is already available:
self.satisfied_by: Optional[Distribution] = None
# Whether the installation process should try to uninstall an existing
# distribution before installing this requirement.
self.should_reinstall = False
@@ -214,7 +242,7 @@ class InstallRequirement:
def name(self) -> Optional[str]:
if self.req is None:
return None
return self.req.name
return pkg_resources.safe_name(self.req.name)
@functools.lru_cache() # use cached_property in python 3.8+
def supports_pyproject_editable(self) -> bool:
@@ -387,24 +415,32 @@ class InstallRequirement:
"""
if self.req is None:
return
existing_dist = get_default_environment().get_distribution(self.req.name)
existing_dist = get_distribution(self.req.name)
if not existing_dist:
return
version_compatible = self.req.specifier.contains(
existing_dist.version,
prereleases=True,
# pkg_resouces may contain a different copy of packaging.version from
# pip in if the downstream distributor does a poor job debundling pip.
# We avoid existing_dist.parsed_version and let SpecifierSet.contains
# parses the version instead.
existing_version = existing_dist.version
version_compatible = (
existing_version is not None
and self.req.specifier.contains(existing_version, prereleases=True)
)
if not version_compatible:
self.satisfied_by = None
if use_user_site:
if existing_dist.in_usersite:
if dist_in_usersite(existing_dist):
self.should_reinstall = True
elif running_under_virtualenv() and existing_dist.in_site_packages:
elif running_under_virtualenv() and dist_in_site_packages(
existing_dist
):
raise InstallationError(
f"Will not install to the user site because it will "
f"lack sys.path precedence to {existing_dist.raw_name} "
f"in {existing_dist.location}"
"Will not install to the user site because it will "
"lack sys.path precedence to {} in {}".format(
existing_dist.project_name, existing_dist.location
)
)
else:
self.should_reinstall = True
@@ -504,7 +540,6 @@ class InstallRequirement:
Under legacy processing, call setup.py egg-info.
"""
assert self.source_dir
details = self.name or f"from {self.link}"
if self.use_pep517:
assert self.pep517_backend is not None
@@ -516,13 +551,11 @@ class InstallRequirement:
self.metadata_directory = generate_editable_metadata(
build_env=self.build_env,
backend=self.pep517_backend,
details=details,
)
else:
self.metadata_directory = generate_metadata(
build_env=self.build_env,
backend=self.pep517_backend,
details=details,
)
else:
self.metadata_directory = generate_metadata_legacy(
@@ -530,7 +563,7 @@ class InstallRequirement:
setup_py_path=self.setup_py_path,
source_dir=self.unpacked_source_directory,
isolated=self.isolated,
details=details,
details=self.name or f"from {self.link}",
)
# Act on the newly generated metadata, based on the name and version.
@@ -544,12 +577,12 @@ class InstallRequirement:
@property
def metadata(self) -> Any:
if not hasattr(self, "_metadata"):
self._metadata = self.get_dist().metadata
self._metadata = get_metadata(self.get_dist())
return self._metadata
def get_dist(self) -> BaseDistribution:
return get_directory_distribution(self.metadata_directory)
def get_dist(self) -> Distribution:
return _get_dist(self.metadata_directory)
def assert_source_matches_version(self) -> None:
assert self.source_dir
@@ -609,7 +642,7 @@ class InstallRequirement:
# So here, if it's neither a path nor a valid VCS URL, it's a bug.
assert vcs_backend, f"Unsupported VCS URL {self.link.url}"
hidden_url = hide_url(self.link.url)
vcs_backend.obtain(self.source_dir, url=hidden_url, verbosity=0)
vcs_backend.obtain(self.source_dir, url=hidden_url)
# Top-level Actions
def uninstall(
@@ -628,7 +661,7 @@ class InstallRequirement:
"""
assert self.req
dist = get_default_environment().get_distribution(self.req.name)
dist = get_distribution(self.req.name)
if not dist:
logger.warning("Skipping %s as it is not installed.", self.name)
return None
@@ -808,7 +841,7 @@ class InstallRequirement:
)
except LegacyInstallFailure as exc:
self.install_succeeded = False
raise exc
raise exc.__cause__
except Exception:
self.install_succeeded = True
raise
@@ -1,3 +1,4 @@
import csv
import functools
import os
import sys
@@ -5,33 +6,47 @@ import sysconfig
from importlib.util import cache_from_source
from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple
from pip._vendor import pkg_resources
from pip._vendor.pkg_resources import Distribution
from pip._internal.exceptions import UninstallationError
from pip._internal.locations import get_bin_prefix, get_bin_user
from pip._internal.metadata import BaseDistribution
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.egg_link import egg_link_path_from_location
from pip._internal.utils.logging import getLogger, indent_log
from pip._internal.utils.misc import ask, is_local, normalize_path, renames, rmtree
from pip._internal.utils.misc import (
ask,
dist_in_usersite,
dist_is_local,
is_local,
normalize_path,
renames,
rmtree,
)
from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory
logger = getLogger(__name__)
def _script_names(bin_dir: str, script_name: str, is_gui: bool) -> Iterator[str]:
def _script_names(dist: Distribution, script_name: str, is_gui: bool) -> List[str]:
"""Create the fully qualified name of the files created by
{console,gui}_scripts for the given ``dist``.
Returns the list of file names
"""
exe_name = os.path.join(bin_dir, script_name)
yield exe_name
if not WINDOWS:
return
yield f"{exe_name}.exe"
yield f"{exe_name}.exe.manifest"
if is_gui:
yield f"{exe_name}-script.pyw"
if dist_in_usersite(dist):
bin_dir = get_bin_user()
else:
yield f"{exe_name}-script.py"
bin_dir = get_bin_prefix()
exe_name = os.path.join(bin_dir, script_name)
paths_to_remove = [exe_name]
if WINDOWS:
paths_to_remove.append(exe_name + ".exe")
paths_to_remove.append(exe_name + ".exe.manifest")
if is_gui:
paths_to_remove.append(exe_name + "-script.pyw")
else:
paths_to_remove.append(exe_name + "-script.py")
return paths_to_remove
def _unique(fn: Callable[..., Iterator[Any]]) -> Callable[..., Iterator[Any]]:
@@ -47,7 +62,7 @@ def _unique(fn: Callable[..., Iterator[Any]]) -> Callable[..., Iterator[Any]]:
@_unique
def uninstallation_paths(dist: BaseDistribution) -> Iterator[str]:
def uninstallation_paths(dist: Distribution) -> Iterator[str]:
"""
Yield all the uninstallation paths for dist based on RECORD-without-.py[co]
@@ -61,25 +76,25 @@ def uninstallation_paths(dist: BaseDistribution) -> Iterator[str]:
https://packaging.python.org/specifications/recording-installed-packages/
"""
location = dist.location
assert location is not None, "not installed"
entries = dist.iter_declared_entries()
if entries is None:
try:
r = csv.reader(dist.get_metadata_lines("RECORD"))
except FileNotFoundError as missing_record_exception:
msg = "Cannot uninstall {dist}, RECORD file not found.".format(dist=dist)
installer = dist.installer
if not installer or installer == "pip":
dep = "{}=={}".format(dist.raw_name, dist.version)
try:
installer = next(dist.get_metadata_lines("INSTALLER"))
if not installer or installer == "pip":
raise ValueError()
except (OSError, StopIteration, ValueError):
dep = "{}=={}".format(dist.project_name, dist.version)
msg += (
" You might be able to recover from this via: "
"'pip install --force-reinstall --no-deps {}'.".format(dep)
)
else:
msg += " Hint: The package was installed by {}.".format(installer)
raise UninstallationError(msg)
for entry in entries:
path = os.path.join(location, entry)
raise UninstallationError(msg) from missing_record_exception
for row in r:
path = os.path.join(dist.location, row[0])
yield path
if path.endswith(".py"):
dn, fn = os.path.split(path)
@@ -302,11 +317,11 @@ class UninstallPathSet:
"""A set of file paths to be removed in the uninstallation of a
requirement."""
def __init__(self, dist: BaseDistribution) -> None:
self._paths: Set[str] = set()
def __init__(self, dist: Distribution) -> None:
self.paths: Set[str] = set()
self._refuse: Set[str] = set()
self._pth: Dict[str, UninstallPthEntries] = {}
self._dist = dist
self.pth: Dict[str, UninstallPthEntries] = {}
self.dist = dist
self._moved_paths = StashedUninstallPathSet()
def _permitted(self, path: str) -> bool:
@@ -327,7 +342,7 @@ class UninstallPathSet:
if not os.path.exists(path):
return
if self._permitted(path):
self._paths.add(path)
self.paths.add(path)
else:
self._refuse.add(path)
@@ -339,37 +354,37 @@ class UninstallPathSet:
def add_pth(self, pth_file: str, entry: str) -> None:
pth_file = normalize_path(pth_file)
if self._permitted(pth_file):
if pth_file not in self._pth:
self._pth[pth_file] = UninstallPthEntries(pth_file)
self._pth[pth_file].add(entry)
if pth_file not in self.pth:
self.pth[pth_file] = UninstallPthEntries(pth_file)
self.pth[pth_file].add(entry)
else:
self._refuse.add(pth_file)
def remove(self, auto_confirm: bool = False, verbose: bool = False) -> None:
"""Remove paths in ``self._paths`` with confirmation (unless
"""Remove paths in ``self.paths`` with confirmation (unless
``auto_confirm`` is True)."""
if not self._paths:
if not self.paths:
logger.info(
"Can't uninstall '%s'. No files were found to uninstall.",
self._dist.raw_name,
self.dist.project_name,
)
return
dist_name_version = f"{self._dist.raw_name}-{self._dist.version}"
dist_name_version = self.dist.project_name + "-" + self.dist.version
logger.info("Uninstalling %s:", dist_name_version)
with indent_log():
if auto_confirm or self._allowed_to_proceed(verbose):
moved = self._moved_paths
for_rename = compress_for_rename(self._paths)
for_rename = compress_for_rename(self.paths)
for path in sorted(compact(for_rename)):
moved.stash(path)
logger.verbose("Removing file or directory %s", path)
for pth in self._pth.values():
for pth in self.pth.values():
pth.remove()
logger.info("Successfully uninstalled %s", dist_name_version)
@@ -387,18 +402,18 @@ class UninstallPathSet:
logger.info(path)
if not verbose:
will_remove, will_skip = compress_for_output_listing(self._paths)
will_remove, will_skip = compress_for_output_listing(self.paths)
else:
# In verbose mode, display all the files that are going to be
# deleted.
will_remove = set(self._paths)
will_remove = set(self.paths)
will_skip = set()
_display("Would remove:", will_remove)
_display("Would not remove (might be manually added):", will_skip)
_display("Would not remove (outside of prefix):", self._refuse)
if verbose:
_display("Will actually move:", compress_for_rename(self._paths))
_display("Will actually move:", compress_for_rename(self.paths))
return ask("Proceed (Y/n)? ", ("y", "n", "")) != "n"
@@ -407,12 +422,12 @@ class UninstallPathSet:
if not self._moved_paths.can_rollback:
logger.error(
"Can't roll back %s; was not uninstalled",
self._dist.raw_name,
self.dist.project_name,
)
return
logger.info("Rolling back uninstall of %s", self._dist.raw_name)
logger.info("Rolling back uninstall of %s", self.dist.project_name)
self._moved_paths.rollback()
for pth in self._pth.values():
for pth in self.pth.values():
pth.rollback()
def commit(self) -> None:
@@ -420,156 +435,141 @@ class UninstallPathSet:
self._moved_paths.commit()
@classmethod
def from_dist(cls, dist: BaseDistribution) -> "UninstallPathSet":
dist_location = dist.location
info_location = dist.info_location
if dist_location is None:
logger.info(
"Not uninstalling %s since it is not installed",
dist.canonical_name,
)
return cls(dist)
normalized_dist_location = normalize_path(dist_location)
if not dist.local:
def from_dist(cls, dist: Distribution) -> "UninstallPathSet":
dist_path = normalize_path(dist.location)
if not dist_is_local(dist):
logger.info(
"Not uninstalling %s at %s, outside environment %s",
dist.canonical_name,
normalized_dist_location,
dist.key,
dist_path,
sys.prefix,
)
return cls(dist)
if normalized_dist_location in {
if dist_path in {
p
for p in {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")}
if p
}:
logger.info(
"Not uninstalling %s at %s, as it is in the standard library.",
dist.canonical_name,
normalized_dist_location,
dist.key,
dist_path,
)
return cls(dist)
paths_to_remove = cls(dist)
develop_egg_link = egg_link_path_from_location(dist.raw_name)
# Distribution is installed with metadata in a "flat" .egg-info
# directory. This means it is not a modern .dist-info installation, an
# egg, or legacy editable.
setuptools_flat_installation = (
dist.installed_with_setuptools_egg_info
and info_location is not None
and os.path.exists(info_location)
# If dist is editable and the location points to a ``.egg-info``,
# we are in fact in the legacy editable case.
and not info_location.endswith(f"{dist.setuptools_filename}.egg-info")
develop_egg_link = egg_link_path_from_location(dist.project_name)
develop_egg_link_egg_info = "{}.egg-info".format(
pkg_resources.to_filename(dist.project_name)
)
egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info)
# Special case for distutils installed package
distutils_egg_info = getattr(dist._provider, "path", None)
# Uninstall cases order do matter as in the case of 2 installs of the
# same package, pip needs to uninstall the currently detected version
if setuptools_flat_installation:
if info_location is not None:
paths_to_remove.add(info_location)
installed_files = dist.iter_declared_entries()
if installed_files is not None:
for installed_file in installed_files:
paths_to_remove.add(os.path.join(dist_location, installed_file))
if (
egg_info_exists
and dist.egg_info.endswith(".egg-info")
and not dist.egg_info.endswith(develop_egg_link_egg_info)
):
# if dist.egg_info.endswith(develop_egg_link_egg_info), we
# are in fact in the develop_egg_link case
paths_to_remove.add(dist.egg_info)
if dist.has_metadata("installed-files.txt"):
for installed_file in dist.get_metadata(
"installed-files.txt"
).splitlines():
path = os.path.normpath(os.path.join(dist.egg_info, installed_file))
paths_to_remove.add(path)
# FIXME: need a test for this elif block
# occurs with --single-version-externally-managed/--record outside
# of pip
elif dist.is_file("top_level.txt"):
try:
namespace_packages = dist.read_text("namespace_packages.txt")
except FileNotFoundError:
namespaces = []
elif dist.has_metadata("top_level.txt"):
if dist.has_metadata("namespace_packages.txt"):
namespaces = dist.get_metadata("namespace_packages.txt")
else:
namespaces = namespace_packages.splitlines(keepends=False)
namespaces = []
for top_level_pkg in [
p
for p in dist.read_text("top_level.txt").splitlines()
for p in dist.get_metadata("top_level.txt").splitlines()
if p and p not in namespaces
]:
path = os.path.join(dist_location, top_level_pkg)
path = os.path.join(dist.location, top_level_pkg)
paths_to_remove.add(path)
paths_to_remove.add(f"{path}.py")
paths_to_remove.add(f"{path}.pyc")
paths_to_remove.add(f"{path}.pyo")
paths_to_remove.add(path + ".py")
paths_to_remove.add(path + ".pyc")
paths_to_remove.add(path + ".pyo")
elif dist.installed_by_distutils:
elif distutils_egg_info:
raise UninstallationError(
"Cannot uninstall {!r}. It is a distutils installed project "
"and thus we cannot accurately determine which files belong "
"to it which would lead to only a partial uninstall.".format(
dist.raw_name,
dist.project_name,
)
)
elif dist.installed_as_egg:
elif dist.location.endswith(".egg"):
# package installed by easy_install
# We cannot match on dist.egg_name because it can slightly vary
# i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg
paths_to_remove.add(dist_location)
easy_install_egg = os.path.split(dist_location)[1]
paths_to_remove.add(dist.location)
easy_install_egg = os.path.split(dist.location)[1]
easy_install_pth = os.path.join(
os.path.dirname(dist_location),
"easy-install.pth",
os.path.dirname(dist.location), "easy-install.pth"
)
paths_to_remove.add_pth(easy_install_pth, "./" + easy_install_egg)
elif dist.installed_with_dist_info:
elif egg_info_exists and dist.egg_info.endswith(".dist-info"):
for path in uninstallation_paths(dist):
paths_to_remove.add(path)
elif develop_egg_link:
# PEP 660 modern editable is handled in the ``.dist-info`` case
# above, so this only covers the setuptools-style editable.
# develop egg
with open(develop_egg_link) as fh:
link_pointer = os.path.normcase(fh.readline().strip())
assert link_pointer == dist_location, (
f"Egg-link {link_pointer} does not match installed location of "
f"{dist.raw_name} (at {dist_location})"
assert (
link_pointer == dist.location
), "Egg-link {} does not match installed location of {} (at {})".format(
link_pointer, dist.project_name, dist.location
)
paths_to_remove.add(develop_egg_link)
easy_install_pth = os.path.join(
os.path.dirname(develop_egg_link), "easy-install.pth"
)
paths_to_remove.add_pth(easy_install_pth, dist_location)
paths_to_remove.add_pth(easy_install_pth, dist.location)
else:
logger.debug(
"Not sure how to uninstall: %s - Check: %s",
dist,
dist_location,
dist.location,
)
if dist.in_usersite:
bin_dir = get_bin_user()
else:
bin_dir = get_bin_prefix()
# find distutils scripts= scripts
try:
for script in dist.iterdir("scripts"):
paths_to_remove.add(os.path.join(bin_dir, script.name))
if dist.has_metadata("scripts") and dist.metadata_isdir("scripts"):
for script in dist.metadata_listdir("scripts"):
if dist_in_usersite(dist):
bin_dir = get_bin_user()
else:
bin_dir = get_bin_prefix()
paths_to_remove.add(os.path.join(bin_dir, script))
if WINDOWS:
paths_to_remove.add(os.path.join(bin_dir, f"{script.name}.bat"))
except (FileNotFoundError, NotADirectoryError):
pass
paths_to_remove.add(os.path.join(bin_dir, script) + ".bat")
# find console_scripts and gui_scripts
def iter_scripts_to_remove(
dist: BaseDistribution,
bin_dir: str,
) -> Iterator[str]:
for entry_point in dist.iter_entry_points():
if entry_point.group == "console_scripts":
yield from _script_names(bin_dir, entry_point.name, False)
elif entry_point.group == "gui_scripts":
yield from _script_names(bin_dir, entry_point.name, True)
# find console_scripts
_scripts_to_remove = []
console_scripts = dist.get_entry_map(group="console_scripts")
for name in console_scripts.keys():
_scripts_to_remove.extend(_script_names(dist, name, False))
# find gui_scripts
gui_scripts = dist.get_entry_map(group="gui_scripts")
for name in gui_scripts.keys():
_scripts_to_remove.extend(_script_names(dist, name, True))
for s in iter_scripts_to_remove(dist, bin_dir):
for s in _scripts_to_remove:
paths_to_remove.add(s)
return paths_to_remove
@@ -43,7 +43,7 @@ from pip._internal.req.req_set import RequirementSet
from pip._internal.resolution.base import BaseResolver, InstallRequirementProvider
from pip._internal.utils.compatibility_tags import get_supported
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import normalize_version_info
from pip._internal.utils.misc import dist_in_usersite, normalize_version_info
from pip._internal.utils.packaging import check_requires_python
logger = logging.getLogger(__name__)
@@ -203,7 +203,7 @@ class Resolver(BaseResolver):
"""
# Don't uninstall the conflict if doing a user install and the
# conflict is not a user install.
if not self.use_user_site or req.satisfied_by.in_usersite:
if not self.use_user_site or dist_in_usersite(req.satisfied_by):
req.should_reinstall = True
req.satisfied_by = None
@@ -5,11 +5,7 @@ from typing import TYPE_CHECKING, Any, FrozenSet, Iterable, Optional, Tuple, Uni
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._internal.exceptions import (
HashError,
InstallationSubprocessError,
MetadataInconsistent,
)
from pip._internal.exceptions import HashError, MetadataInconsistent
from pip._internal.metadata import BaseDistribution
from pip._internal.models.link import Link, links_equivalent
from pip._internal.models.wheel import Wheel
@@ -98,6 +94,8 @@ def make_install_req_from_editable(
def _make_install_req_from_dist(
dist: BaseDistribution, template: InstallRequirement
) -> InstallRequirement:
from pip._internal.metadata.pkg_resources import Distribution as _Dist
if template.req:
line = str(template.req)
elif template.link:
@@ -117,7 +115,7 @@ def _make_install_req_from_dist(
hashes=template.hash_options,
),
)
ireq.satisfied_by = dist
ireq.satisfied_by = cast(_Dist, dist)._dist
return ireq
@@ -231,11 +229,6 @@ class _InstallRequirementBackedCandidate(Candidate):
# offending line to the user.
e.req = self._ireq
raise
except InstallationSubprocessError as exc:
# The output has been presented already, so don't duplicate it.
exc.context = "See above for output."
raise
self._check_metadata_consistency(dist)
return dist
@@ -97,7 +97,6 @@ class Factory:
force_reinstall: bool,
ignore_installed: bool,
ignore_requires_python: bool,
suppress_build_failures: bool,
py_version_info: Optional[Tuple[int, ...]] = None,
) -> None:
self._finder = finder
@@ -108,7 +107,6 @@ class Factory:
self._use_user_site = use_user_site
self._force_reinstall = force_reinstall
self._ignore_requires_python = ignore_requires_python
self._suppress_build_failures = suppress_build_failures
self._build_failures: Cache[InstallationError] = {}
self._link_candidate_cache: Cache[LinkCandidate] = {}
@@ -192,22 +190,10 @@ class Factory:
name=name,
version=version,
)
except MetadataInconsistent as e:
logger.info(
"Discarding [blue underline]%s[/]: [yellow]%s[reset]",
link,
e,
extra={"markup": True},
)
except (InstallationSubprocessError, MetadataInconsistent) as e:
logger.warning("Discarding %s. %s", link, e)
self._build_failures[link] = e
return None
except InstallationSubprocessError as e:
if not self._suppress_build_failures:
raise
logger.warning("Discarding %s due to build failure: %s", link, e)
self._build_failures[link] = e
return None
base: BaseCandidate = self._editable_candidate_cache[link]
else:
if link not in self._link_candidate_cache:
@@ -219,19 +205,8 @@ class Factory:
name=name,
version=version,
)
except MetadataInconsistent as e:
logger.info(
"Discarding [blue underline]%s[/]: [yellow]%s[reset]",
link,
e,
extra={"markup": True},
)
self._build_failures[link] = e
return None
except InstallationSubprocessError as e:
if not self._suppress_build_failures:
raise
logger.warning("Discarding %s due to build failure: %s", link, e)
except (InstallationSubprocessError, MetadataInconsistent) as e:
logger.warning("Discarding %s. %s", link, e)
self._build_failures[link] = e
return None
base = self._link_candidate_cache[link]
@@ -285,7 +260,7 @@ class Factory:
extras=extras,
template=template,
)
# The candidate is a known incompatibility. Don't use it.
# The candidate is a known incompatiblity. Don't use it.
if id(candidate) in incompatible_ids:
return None
return candidate
@@ -298,27 +273,14 @@ class Factory:
)
icans = list(result.iter_applicable())
# PEP 592: Yanked releases are ignored unless the specifier
# explicitly pins a version (via '==' or '===') that can be
# solely satisfied by a yanked release.
# PEP 592: Yanked releases must be ignored unless only yanked
# releases can satisfy the version range. So if this is false,
# all yanked icans need to be skipped.
all_yanked = all(ican.link.is_yanked for ican in icans)
def is_pinned(specifier: SpecifierSet) -> bool:
for sp in specifier:
if sp.operator == "===":
return True
if sp.operator != "==":
continue
if sp.version.endswith(".*"):
continue
return True
return False
pinned = is_pinned(specifier)
# PackageFinder returns earlier versions first, so we reverse.
for ican in reversed(icans):
if not (all_yanked and pinned) and ican.link.is_yanked:
if not all_yanked and ican.link.is_yanked:
continue
func = functools.partial(
self._make_candidate_from_link,
@@ -412,7 +374,7 @@ class Factory:
)
# Add explicit candidates from constraints. We only do this if there are
# known ireqs, which represent requirements not already explicit. If
# kown ireqs, which represent requirements not already explicit. If
# there are no ireqs, we're constraining already-explicit requirements,
# which is handled later when we return the explicit candidates.
if ireqs:
@@ -653,7 +615,7 @@ class Factory:
]
if requires_python_causes:
# The comprehension above makes sure all Requirement instances are
# RequiresPythonRequirement, so let's cast for convenience.
# RequiresPythonRequirement, so let's cast for convinience.
return self._report_requires_python_error(
cast("Sequence[ConflictCause]", requires_python_causes),
)
@@ -734,6 +696,6 @@ class Factory:
return DistributionNotFound(
"ResolutionImpossible: for help visit "
"https://pip.pypa.io/en/latest/topics/dependency-resolution/"
"#dealing-with-dependency-conflicts"
"https://pip.pypa.io/en/latest/user_guide/"
"#fixing-conflicting-dependencies"
)
@@ -1,15 +1,6 @@
import collections
import math
from typing import (
TYPE_CHECKING,
Dict,
Iterable,
Iterator,
Mapping,
Sequence,
TypeVar,
Union,
)
from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Mapping, Sequence, Union
from pip._vendor.resolvelib.providers import AbstractProvider
@@ -46,35 +37,6 @@ else:
# services to those objects (access to pip's finder and preparer).
D = TypeVar("D")
V = TypeVar("V")
def _get_with_identifier(
mapping: Mapping[str, V],
identifier: str,
default: D,
) -> Union[D, V]:
"""Get item from a package name lookup mapping with a resolver identifier.
This extra logic is needed when the target mapping is keyed by package
name, which cannot be directly looked up with an identifier (which may
contain requested extras). Additional logic is added to also look up a value
by "cleaning up" the extras from the identifier.
"""
if identifier in mapping:
return mapping[identifier]
# HACK: Theoretically we should check whether this identifier is a valid
# "NAME[EXTRAS]" format, and parse out the name part with packaging or
# some regular expression. But since pip's resolver only spits out three
# kinds of identifiers: normalized PEP 503 names, normalized names plus
# extras, and Requires-Python, we can cheat a bit here.
name, open_bracket, _ = identifier.partition("[")
if open_bracket and name in mapping:
return mapping[name]
return default
class PipProvider(_ProviderBase):
"""Pip's provider implementation for resolvelib.
@@ -167,7 +129,7 @@ class PipProvider(_ProviderBase):
# (Most projects specify it only to request for an installer feature,
# which does not work, but that's another topic.) Intentionally
# delaying Setuptools helps reduce branches the resolver has to check.
# This serves as a temporary fix for issues like "apache-airflow[all]"
# This serves as a temporary fix for issues like "apache-airlfow[all]"
# while we work on "proper" branch pruning techniques.
delay_this = identifier == "setuptools"
@@ -188,13 +150,28 @@ class PipProvider(_ProviderBase):
identifier,
)
def _get_constraint(self, identifier: str) -> Constraint:
if identifier in self._constraints:
return self._constraints[identifier]
# HACK: Theoratically we should check whether this identifier is a valid
# "NAME[EXTRAS]" format, and parse out the name part with packaging or
# some regular expression. But since pip's resolver only spits out
# three kinds of identifiers: normalized PEP 503 names, normalized names
# plus extras, and Requires-Python, we can cheat a bit here.
name, open_bracket, _ = identifier.partition("[")
if open_bracket and name in self._constraints:
return self._constraints[name]
return Constraint.empty()
def find_matches(
self,
identifier: str,
requirements: Mapping[str, Iterator[Requirement]],
incompatibilities: Mapping[str, Iterator[Candidate]],
) -> Iterable[Candidate]:
def _eligible_for_upgrade(identifier: str) -> bool:
def _eligible_for_upgrade(name: str) -> bool:
"""Are upgrades allowed for this project?
This checks the upgrade strategy, and whether the project was one
@@ -208,23 +185,13 @@ class PipProvider(_ProviderBase):
if self._upgrade_strategy == "eager":
return True
elif self._upgrade_strategy == "only-if-needed":
user_order = _get_with_identifier(
self._user_requested,
identifier,
default=None,
)
return user_order is not None
return name in self._user_requested
return False
constraint = _get_with_identifier(
self._constraints,
identifier,
default=Constraint.empty(),
)
return self._factory.find_candidates(
identifier=identifier,
requirements=requirements,
constraint=constraint,
constraint=self._get_constraint(identifier),
prefers_installed=(not _eligible_for_upgrade(identifier)),
incompatibilities=incompatibilities,
)
@@ -21,12 +21,12 @@ class ExplicitRequirement(Requirement):
@property
def project_name(self) -> NormalizedName:
# No need to canonicalize - the candidate did this
# No need to canonicalise - the candidate did this
return self.candidate.project_name
@property
def name(self) -> str:
# No need to canonicalize - the candidate did this
# No need to canonicalise - the candidate did this
return self.candidate.name
def format_for_error(self) -> str:
@@ -47,7 +47,6 @@ class Resolver(BaseResolver):
ignore_requires_python: bool,
force_reinstall: bool,
upgrade_strategy: str,
suppress_build_failures: bool,
py_version_info: Optional[Tuple[int, ...]] = None,
):
super().__init__()
@@ -62,7 +61,6 @@ class Resolver(BaseResolver):
force_reinstall=force_reinstall,
ignore_installed=ignore_installed,
ignore_requires_python=ignore_requires_python,
suppress_build_failures=suppress_build_failures,
py_version_info=py_version_info,
)
self.ignore_dependencies = ignore_dependencies
@@ -173,19 +171,17 @@ class Resolver(BaseResolver):
get installed one-by-one.
The current implementation creates a topological ordering of the
dependency graph, giving more weight to packages with less
or no dependencies, while breaking any cycles in the graph at
arbitrary points. We make no guarantees about where the cycle
would be broken, other than it *would* be broken.
dependency graph, while breaking any cycles in the graph at arbitrary
points. We make no guarantees about where the cycle would be broken,
other than they would be broken.
"""
assert self._result is not None, "must call resolve() first"
if not req_set.requirements:
# Nothing is left to install, so we do not need an order.
return []
graph = self._result.graph
weights = get_topological_weights(graph, set(req_set.requirements.keys()))
weights = get_topological_weights(
graph,
expected_node_count=len(self._result.mapping) + 1,
)
sorted_items = sorted(
req_set.requirements.items(),
@@ -196,32 +192,23 @@ class Resolver(BaseResolver):
def get_topological_weights(
graph: "DirectedGraph[Optional[str]]", requirement_keys: Set[str]
graph: "DirectedGraph[Optional[str]]", expected_node_count: int
) -> Dict[Optional[str], int]:
"""Assign weights to each node based on how "deep" they are.
This implementation may change at any point in the future without prior
notice.
We first simplify the dependency graph by pruning any leaves and giving them
the highest weight: a package without any dependencies should be installed
first. This is done again and again in the same way, giving ever less weight
to the newly found leaves. The loop stops when no leaves are left: all
remaining packages have at least one dependency left in the graph.
Then we continue with the remaining graph, by taking the length for the
longest path to any node from root, ignoring any paths that contain a single
node twice (i.e. cycles). This is done through a depth-first search through
the graph, while keeping track of the path to the node.
We take the length for the longest path to any node from root, ignoring any
paths that contain a single node twice (i.e. cycles). This is done through
a depth-first search through the graph, while keeping track of the path to
the node.
Cycles in the graph result would result in node being revisited while also
being on its own path. In this case, take no action. This helps ensure we
being it's own path. In this case, take no action. This helps ensure we
don't get stuck in a cycle.
When assigning weight, the longer path (i.e. larger length) is preferred.
We are only interested in the weights of packages that are in the
requirement_keys.
"""
path: Set[Optional[str]] = set()
weights: Dict[Optional[str], int] = {}
@@ -237,49 +224,15 @@ def get_topological_weights(
visit(child)
path.remove(node)
if node not in requirement_keys:
return
last_known_parent_count = weights.get(node, 0)
weights[node] = max(last_known_parent_count, len(path))
# Simplify the graph, pruning leaves that have no dependencies.
# This is needed for large graphs (say over 200 packages) because the
# `visit` function is exponentially slower then, taking minutes.
# See https://github.com/pypa/pip/issues/10557
# We will loop until we explicitly break the loop.
while True:
leaves = set()
for key in graph:
if key is None:
continue
for _child in graph.iter_children(key):
# This means we have at least one child
break
else:
# No child.
leaves.add(key)
if not leaves:
# We are done simplifying.
break
# Calculate the weight for the leaves.
weight = len(graph) - 1
for leaf in leaves:
if leaf not in requirement_keys:
continue
weights[leaf] = weight
# Remove the leaves from the graph, making it simpler.
for leaf in leaves:
graph.remove(leaf)
# Visit the remaining graph.
# `None` is guaranteed to be the root node by resolvelib.
visit(None)
# Sanity check: all requirement keys should be in the weights,
# and no other keys should be in the weights.
difference = set(weights.keys()).difference(requirement_keys)
assert not difference, difference
# Sanity checks
assert weights[None] == 0
assert len(weights) == expected_node_count
return weights
@@ -141,9 +141,6 @@ def pip_self_version_check(session: PipSession, options: optparse.Values) -> Non
finder = PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
use_deprecated_html5lib=(
"html5lib" in options.deprecated_features_enabled
),
)
best_candidate = finder.find_best_candidate("pip").best_candidate
if best_candidate is None:
@@ -168,11 +165,7 @@ def pip_self_version_check(session: PipSession, options: optparse.Values) -> Non
# We cannot tell how the current pip is available in the current
# command context, so be pragmatic here and suggest the command
# that's always available. This does not accommodate spaces in
# `sys.executable` on purpose as it is not possible to do it
# correctly without knowing the user's shell. Thus,
# it won't be done until possible through the standard library.
# Do not be tempted to use the undocumented subprocess.list2cmdline.
# It is considered an internal implementation detail for a reason.
# `sys.executable`.
pip_cmd = f"{sys.executable} -m pip"
logger.warning(
"You are using pip version %s; however, version %s is "
@@ -4,28 +4,28 @@ import logging
import logging.handlers
import os
import sys
import threading
from dataclasses import dataclass
from logging import Filter
from typing import IO, Any, ClassVar, Iterator, List, Optional, TextIO, Type
from typing import IO, Any, Callable, Iterator, Optional, TextIO, Type, cast
from pip._vendor.rich.console import (
Console,
ConsoleOptions,
ConsoleRenderable,
RenderResult,
)
from pip._vendor.rich.highlighter import NullHighlighter
from pip._vendor.rich.logging import RichHandler
from pip._vendor.rich.segment import Segment
from pip._vendor.rich.style import Style
from pip._internal.exceptions import DiagnosticPipError
from pip._internal.utils._log import VERBOSE, getLogger
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX
from pip._internal.utils.misc import ensure_dir
try:
import threading
except ImportError:
import dummy_threading as threading # type: ignore
try:
from pip._vendor import colorama
# Lots of different errors can come from this, including SystemError and
# ImportError.
except Exception:
colorama = None
_log_state = threading.local()
subprocess_logger = getLogger("pip.subprocessor")
@@ -119,63 +119,78 @@ class IndentingFormatter(logging.Formatter):
return formatted
@dataclass
class IndentedRenderable:
renderable: ConsoleRenderable
indent: int
def _color_wrap(*colors: str) -> Callable[[str], str]:
def wrapped(inp: str) -> str:
return "".join(list(colors) + [inp, colorama.Style.RESET_ALL])
def __rich_console__(
self, console: Console, options: ConsoleOptions
) -> RenderResult:
segments = console.render(self.renderable, options)
lines = Segment.split_lines(segments)
for line in lines:
yield Segment(" " * self.indent)
yield from line
yield Segment("\n")
return wrapped
class RichPipStreamHandler(RichHandler):
KEYWORDS: ClassVar[Optional[List[str]]] = []
class ColorizedStreamHandler(logging.StreamHandler):
def __init__(self, stream: Optional[TextIO], no_color: bool) -> None:
super().__init__(
console=Console(file=stream, no_color=no_color, soft_wrap=True),
show_time=False,
show_level=False,
show_path=False,
highlighter=NullHighlighter(),
# Don't build up a list of colors if we don't have colorama
if colorama:
COLORS = [
# This needs to be in order from highest logging level to lowest.
(logging.ERROR, _color_wrap(colorama.Fore.RED)),
(logging.WARNING, _color_wrap(colorama.Fore.YELLOW)),
]
else:
COLORS = []
def __init__(self, stream: Optional[TextIO] = None, no_color: bool = None) -> None:
super().__init__(stream)
self._no_color = no_color
if WINDOWS and colorama:
self.stream = colorama.AnsiToWin32(self.stream)
def _using_stdout(self) -> bool:
"""
Return whether the handler is using sys.stdout.
"""
if WINDOWS and colorama:
# Then self.stream is an AnsiToWin32 object.
stream = cast(colorama.AnsiToWin32, self.stream)
return stream.wrapped is sys.stdout
return self.stream is sys.stdout
def should_color(self) -> bool:
# Don't colorize things if we do not have colorama or if told not to
if not colorama or self._no_color:
return False
real_stream = (
self.stream
if not isinstance(self.stream, colorama.AnsiToWin32)
else self.stream.wrapped
)
# Our custom override on Rich's logger, to make things work as we need them to.
def emit(self, record: logging.LogRecord) -> None:
style: Optional[Style] = None
# If the stream is a tty we should color it
if hasattr(real_stream, "isatty") and real_stream.isatty():
return True
# If we are given a diagnostic error to present, present it with indentation.
if record.msg == "[present-diagnostic] %s" and len(record.args) == 1:
diagnostic_error: DiagnosticPipError = record.args[0] # type: ignore[index]
assert isinstance(diagnostic_error, DiagnosticPipError)
# If we have an ANSI term we should color it
if os.environ.get("TERM") == "ANSI":
return True
renderable: ConsoleRenderable = IndentedRenderable(
diagnostic_error, indent=get_indentation()
)
else:
message = self.format(record)
renderable = self.render_message(record, message)
if record.levelno is not None:
if record.levelno >= logging.ERROR:
style = Style(color="red")
elif record.levelno >= logging.WARNING:
style = Style(color="yellow")
# If anything else we should not color it
return False
try:
self.console.print(renderable, overflow="ignore", crop=False, style=style)
except Exception:
self.handleError(record)
def format(self, record: logging.LogRecord) -> str:
msg = super().format(record)
if self.should_color():
for level, color in self.COLORS:
if record.levelno >= level:
msg = color(msg)
break
return msg
# The logging module says handleError() can be customized.
def handleError(self, record: logging.LogRecord) -> None:
"""Called when logging is unable to log some output."""
exc_class, exc = sys.exc_info()[:2]
# If a broken pipe occurred while calling write() or flush() on the
# stdout stream in logging's Handler.emit(), then raise our special
@@ -184,7 +199,7 @@ class RichPipStreamHandler(RichHandler):
if (
exc_class
and exc
and self.console.file is sys.stdout
and self._using_stdout()
and _is_broken_pipe_error(exc_class, exc)
):
raise BrokenStdoutLoggingError()
@@ -260,7 +275,7 @@ def setup_logging(verbosity: int, no_color: bool, user_log_file: Optional[str])
"stderr": "ext://sys.stderr",
}
handler_classes = {
"stream": "pip._internal.utils.logging.RichPipStreamHandler",
"stream": "pip._internal.utils.logging.ColorizedStreamHandler",
"file": "pip._internal.utils.logging.BetterRotatingFileHandler",
}
handlers = ["console", "console_errors", "console_subprocess"] + (
@@ -318,8 +333,8 @@ def setup_logging(verbosity: int, no_color: bool, user_log_file: Optional[str])
"console_subprocess": {
"level": level,
"class": handler_classes["stream"],
"stream": log_streams["stderr"],
"no_color": no_color,
"stream": log_streams["stderr"],
"filters": ["restrict_to_subprocess"],
"formatter": "indent",
},
@@ -32,12 +32,14 @@ from typing import (
cast,
)
from pip._vendor.pkg_resources import Distribution
from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
from pip import __version__
from pip._internal.exceptions import CommandError
from pip._internal.locations import get_major_minor_version
from pip._internal.locations import get_major_minor_version, site_packages, user_site
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.egg_link import egg_link_path_from_location
from pip._internal.utils.virtualenv import running_under_virtualenv
__all__ = [
@@ -326,6 +328,64 @@ def is_local(path: str) -> bool:
return path.startswith(normalize_path(sys.prefix))
def dist_is_local(dist: Distribution) -> bool:
"""
Return True if given Distribution object is installed locally
(i.e. within current virtualenv).
Always True if we're not in a virtualenv.
"""
return is_local(dist_location(dist))
def dist_in_usersite(dist: Distribution) -> bool:
"""
Return True if given Distribution is installed in user site.
"""
return dist_location(dist).startswith(normalize_path(user_site))
def dist_in_site_packages(dist: Distribution) -> bool:
"""
Return True if given Distribution is installed in
sysconfig.get_python_lib().
"""
return dist_location(dist).startswith(normalize_path(site_packages))
def get_distribution(req_name: str) -> Optional[Distribution]:
"""Given a requirement name, return the installed Distribution object.
This searches from *all* distributions available in the environment, to
match the behavior of ``pkg_resources.get_distribution()``.
Left for compatibility until direct pkg_resources uses are refactored out.
"""
from pip._internal.metadata import get_default_environment
from pip._internal.metadata.pkg_resources import Distribution as _Dist
dist = get_default_environment().get_distribution(req_name)
if dist is None:
return None
return cast(_Dist, dist)._dist
def dist_location(dist: Distribution) -> str:
"""
Get the site-packages location of this distribution. Generally
this is dist.location, except in the case of develop-installed
packages, where dist.location is the source code location, and we
want to know where the egg-link file is.
The returned location is normalized (in particular, with symlinks removed).
"""
egg_link = egg_link_path_from_location(dist.project_name)
if egg_link:
return normalize_path(egg_link)
return normalize_path(dist.location)
def write_output(msg: Any, *args: Any) -> None:
logger.info(msg, *args)
@@ -1,12 +1,16 @@
import functools
import logging
import re
from typing import NewType, Optional, Tuple, cast
from email.message import Message
from email.parser import FeedParser
from typing import Optional, Tuple
from pip._vendor import pkg_resources
from pip._vendor.packaging import specifiers, version
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.pkg_resources import Distribution
NormalizedExtra = NewType("NormalizedExtra", str)
from pip._internal.exceptions import NoneMetadataError
from pip._internal.utils.misc import display_path
logger = logging.getLogger(__name__)
@@ -34,6 +38,41 @@ def check_requires_python(
return python_version in requires_python_specifier
def get_metadata(dist: Distribution) -> Message:
"""
:raises NoneMetadataError: if the distribution reports `has_metadata()`
True but `get_metadata()` returns None.
"""
metadata_name = "METADATA"
if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata(
metadata_name
):
metadata = dist.get_metadata(metadata_name)
elif dist.has_metadata("PKG-INFO"):
metadata_name = "PKG-INFO"
metadata = dist.get_metadata(metadata_name)
else:
logger.warning("No metadata found in %s", display_path(dist.location))
metadata = ""
if metadata is None:
raise NoneMetadataError(dist, metadata_name)
feed_parser = FeedParser()
# The following line errors out if with a "NoneType" TypeError if
# passed metadata=None.
feed_parser.feed(metadata)
return feed_parser.close()
def get_installer(dist: Distribution) -> str:
if dist.has_metadata("INSTALLER"):
for line in dist.get_metadata_lines("INSTALLER"):
if line.strip():
return line.strip()
return ""
@functools.lru_cache(maxsize=512)
def get_requirement(req_string: str) -> Requirement:
"""Construct a packaging.Requirement object with caching"""
@@ -43,15 +82,3 @@ def get_requirement(req_string: str) -> Requirement:
# minimize repeated parsing of the same string to construct equivalent
# Requirement objects.
return Requirement(req_string)
def safe_extra(extra: str) -> NormalizedExtra:
"""Convert an arbitrary string to a standard 'extra' name
Any runs of non-alphanumeric characters are replaced with a single '_',
and the result is always lowercased.
This function is duplicated from ``pkg_resources``. Note that this is not
the same to either ``canonicalize_name`` or ``_egg_link_name``.
"""
return cast(NormalizedExtra, re.sub("[^A-Za-z0-9.-]+", "_", extra).lower())
@@ -1,49 +1,21 @@
import sys
import textwrap
from typing import List, Optional, Sequence
# Shim to wrap setup.py invocation with setuptools
# Note that __file__ is handled via two {!r} *and* %r, to ensure that paths on
# Windows are correctly handled (it should be "C:\\Users" not "C:\Users").
_SETUPTOOLS_SHIM = textwrap.dedent(
"""
exec(compile('''
# This is <pip-setuptools-caller> -- a caller that pip uses to run setup.py
#
# - It imports setuptools before invoking setup.py, to enable projects that directly
# import from `distutils.core` to work with newer packaging standards.
# - It provides a clear error message when setuptools is not installed.
# - It sets `sys.argv[0]` to the underlying `setup.py`, when invoking `setup.py` so
# setuptools doesn't think the script is `-c`. This avoids the following warning:
# manifest_maker: standard file '-c' not found".
# - It generates a shim setup.py, for handling setup.cfg-only projects.
import os, sys, tokenize
try:
import setuptools
except ImportError as error:
print(
"ERROR: Can not execute `setup.py` since setuptools is not available in "
"the build environment.",
file=sys.stderr,
)
sys.exit(1)
__file__ = %r
sys.argv[0] = __file__
if os.path.exists(__file__):
filename = __file__
with tokenize.open(__file__) as f:
setup_py_code = f.read()
else:
filename = "<auto-generated setuptools caller>"
setup_py_code = "from setuptools import setup; setup()"
exec(compile(setup_py_code, filename, "exec"))
''' % ({!r},), "<pip-setuptools-caller>", "exec"))
"""
).rstrip()
#
# We set sys.argv[0] to the path to the underlying setup.py file so
# setuptools / distutils don't take the path to the setup.py to be "-c" when
# invoking via the shim. This avoids e.g. the following manifest_maker
# warning: "warning: manifest_maker: standard file '-c' not found".
_SETUPTOOLS_SHIM = (
"import io, os, sys, setuptools, tokenize; sys.argv[0] = {0!r}; __file__={0!r};"
"f = getattr(tokenize, 'open', open)(__file__) "
"if os.path.exists(__file__) "
"else io.StringIO('from setuptools import setup; setup()');"
"code = f.read().replace('\\r\\n', '\\n');"
"f.close();"
"exec(compile(code, __file__, 'exec'))"
)
def make_setuptools_shim_args(
@@ -13,8 +13,6 @@ from typing import (
Union,
)
from pip._vendor.rich.markup import escape
from pip._internal.cli.spinners import SpinnerInterface, open_spinner
from pip._internal.exceptions import InstallationSubprocessError
from pip._internal.utils.logging import VERBOSE, subprocess_logger
@@ -29,6 +27,9 @@ if TYPE_CHECKING:
CommandArgs = List[Union[str, HiddenText]]
LOG_DIVIDER = "----------------------------------------"
def make_command(*args: Union[str, HiddenText, CommandArgs]) -> CommandArgs:
"""
Create a CommandArgs object.
@@ -68,19 +69,53 @@ def reveal_command_args(args: Union[List[str], CommandArgs]) -> List[str]:
return [arg.secret if isinstance(arg, HiddenText) else arg for arg in args]
def make_subprocess_output_error(
cmd_args: Union[List[str], CommandArgs],
cwd: Optional[str],
lines: List[str],
exit_status: int,
) -> str:
"""
Create and return the error message to use to log a subprocess error
with command output.
:param lines: A list of lines, each ending with a newline.
"""
command = format_command_args(cmd_args)
# We know the joined output value ends in a newline.
output = "".join(lines)
msg = (
# Use a unicode string to avoid "UnicodeEncodeError: 'ascii'
# codec can't encode character ..." in Python 2 when a format
# argument (e.g. `output`) has a non-ascii character.
"Command errored out with exit status {exit_status}:\n"
" command: {command_display}\n"
" cwd: {cwd_display}\n"
"Complete output ({line_count} lines):\n{output}{divider}"
).format(
exit_status=exit_status,
command_display=command,
cwd_display=cwd,
line_count=len(lines),
output=output,
divider=LOG_DIVIDER,
)
return msg
def call_subprocess(
cmd: Union[List[str], CommandArgs],
show_stdout: bool = False,
cwd: Optional[str] = None,
on_returncode: 'Literal["raise", "warn", "ignore"]' = "raise",
extra_ok_returncodes: Optional[Iterable[int]] = None,
command_desc: Optional[str] = None,
extra_environ: Optional[Mapping[str, Any]] = None,
unset_environ: Optional[Iterable[str]] = None,
spinner: Optional[SpinnerInterface] = None,
log_failed_cmd: Optional[bool] = True,
stdout_only: Optional[bool] = False,
*,
command_desc: str,
) -> str:
"""
Args:
@@ -131,6 +166,9 @@ def call_subprocess(
# and we have a spinner.
use_spinner = not showing_subprocess and spinner is not None
if command_desc is None:
command_desc = format_command_args(cmd)
log_subprocess("Running command %s", command_desc)
env = os.environ.copy()
if extra_environ:
@@ -203,25 +241,17 @@ def call_subprocess(
spinner.finish("done")
if proc_had_error:
if on_returncode == "raise":
error = InstallationSubprocessError(
command_description=command_desc,
exit_code=proc.returncode,
output_lines=all_output if not showing_subprocess else None,
)
if log_failed_cmd:
subprocess_logger.error("[present-diagnostic] %s", error)
subprocess_logger.verbose(
"[bold magenta]full command[/]: [blue]%s[/]",
escape(format_command_args(cmd)),
extra={"markup": True},
if not showing_subprocess and log_failed_cmd:
# Then the subprocess streams haven't been logged to the
# console yet.
msg = make_subprocess_output_error(
cmd_args=cmd,
cwd=cwd,
lines=all_output,
exit_status=proc.returncode,
)
subprocess_logger.verbose(
"[bold magenta]cwd[/]: %s",
escape(cwd or "[inherit]"),
extra={"markup": True},
)
raise error
subprocess_logger.error(msg)
raise InstallationSubprocessError(proc.returncode, command_desc)
elif on_returncode == "warn":
subprocess_logger.warning(
'Command "%s" had error code %s in %s',
@@ -251,7 +281,6 @@ def runner_with_spinner_message(message: str) -> Callable[..., None]:
with open_spinner(message) as spinner:
call_subprocess(
cmd,
command_desc=message,
cwd=cwd,
extra_environ=extra_environ,
spinner=spinner,
@@ -4,12 +4,14 @@
import logging
from email.message import Message
from email.parser import Parser
from typing import Tuple
from typing import Dict, Tuple
from zipfile import BadZipFile, ZipFile
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.pkg_resources import DistInfoDistribution, Distribution
from pip._internal.exceptions import UnsupportedWheel
from pip._internal.utils.pkg_resources import DictMetadata
VERSION_COMPATIBLE = (1, 0)
@@ -17,6 +19,50 @@ VERSION_COMPATIBLE = (1, 0)
logger = logging.getLogger(__name__)
class WheelMetadata(DictMetadata):
"""Metadata provider that maps metadata decoding exceptions to our
internal exception type.
"""
def __init__(self, metadata: Dict[str, bytes], wheel_name: str) -> None:
super().__init__(metadata)
self._wheel_name = wheel_name
def get_metadata(self, name: str) -> str:
try:
return super().get_metadata(name)
except UnicodeDecodeError as e:
# Augment the default error with the origin of the file.
raise UnsupportedWheel(
f"Error decoding metadata for {self._wheel_name}: {e}"
)
def pkg_resources_distribution_for_wheel(
wheel_zip: ZipFile, name: str, location: str
) -> Distribution:
"""Get a pkg_resources distribution given a wheel.
:raises UnsupportedWheel: on any errors
"""
info_dir, _ = parse_wheel(wheel_zip, name)
metadata_files = [p for p in wheel_zip.namelist() if p.startswith(f"{info_dir}/")]
metadata_text: Dict[str, bytes] = {}
for path in metadata_files:
_, metadata_name = path.split("/", 1)
try:
metadata_text[metadata_name] = read_wheel_metadata_file(wheel_zip, path)
except UnsupportedWheel as e:
raise UnsupportedWheel("{} has an invalid wheel, {}".format(name, str(e)))
metadata = WheelMetadata(metadata_text, location)
return DistInfoDistribution(location=location, metadata=metadata, project_name=name)
def parse_wheel(wheel_zip: ZipFile, name: str) -> Tuple[str, Message]:
"""Extract information from the provided wheel, ensuring it meets basic
standards.
@@ -33,9 +33,7 @@ class Bazaar(VersionControl):
def get_base_rev_args(rev: str) -> List[str]:
return ["-r", rev]
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
rev_display = rev_options.to_display()
logger.info(
"Checking out %s%s to %s",
@@ -43,13 +41,7 @@ class Bazaar(VersionControl):
rev_display,
display_path(dest),
)
if verbosity <= 0:
flag = "--quiet"
elif verbosity == 1:
flag = ""
else:
flag = f"-{'v'*verbosity}"
cmd_args = make_command("branch", flag, rev_options.to_args(), url, dest)
cmd_args = make_command("branch", "-q", rev_options.to_args(), url, dest)
self.run_command(cmd_args)
def switch(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
@@ -91,12 +91,7 @@ class Git(VersionControl):
return not is_tag_or_branch
def get_git_version(self) -> Tuple[int, ...]:
version = self.run_command(
["version"],
command_desc="git version",
show_stdout=False,
stdout_only=True,
)
version = self.run_command(["version"], show_stdout=False, stdout_only=True)
match = GIT_VERSION_REGEX.match(version)
if not match:
logger.warning("Can't parse git version: %s", version)
@@ -258,17 +253,9 @@ class Git(VersionControl):
return cls.get_revision(dest) == name
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
rev_display = rev_options.to_display()
logger.info("Cloning %s%s to %s", url, rev_display, display_path(dest))
if verbosity <= 0:
flags: Tuple[str, ...] = ("--quiet",)
elif verbosity == 1:
flags = ()
else:
flags = ("--verbose", "--progress")
if self.get_git_version() >= (2, 17):
# Git added support for partial clone in 2.17
# https://git-scm.com/docs/partial-clone
@@ -277,13 +264,13 @@ class Git(VersionControl):
make_command(
"clone",
"--filter=blob:none",
*flags,
"-q",
url,
dest,
)
)
else:
self.run_command(make_command("clone", *flags, url, dest))
self.run_command(make_command("clone", "-q", url, dest))
if rev_options.rev:
# Then a specific revision was requested.
@@ -1,7 +1,7 @@
import configparser
import logging
import os
from typing import List, Optional, Tuple
from typing import List, Optional
from pip._internal.exceptions import BadCommand, InstallationError
from pip._internal.utils.misc import HiddenText, display_path
@@ -33,9 +33,7 @@ class Mercurial(VersionControl):
def get_base_rev_args(rev: str) -> List[str]:
return [rev]
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
rev_display = rev_options.to_display()
logger.info(
"Cloning hg %s%s to %s",
@@ -43,17 +41,9 @@ class Mercurial(VersionControl):
rev_display,
display_path(dest),
)
if verbosity <= 0:
flags: Tuple[str, ...] = ("--quiet",)
elif verbosity == 1:
flags = ()
elif verbosity == 2:
flags = ("--verbose",)
else:
flags = ("--verbose", "--debug")
self.run_command(make_command("clone", "--noupdate", *flags, url, dest))
self.run_command(make_command("clone", "--noupdate", "-q", url, dest))
self.run_command(
make_command("update", *flags, rev_options.to_args()),
make_command("update", "-q", rev_options.to_args()),
cwd=dest,
)
@@ -277,9 +277,7 @@ class Subversion(VersionControl):
return []
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
rev_display = rev_options.to_display()
logger.info(
"Checking out %s%s to %s",
@@ -287,13 +285,9 @@ class Subversion(VersionControl):
rev_display,
display_path(dest),
)
if verbosity <= 0:
flag = "--quiet"
else:
flag = ""
cmd_args = make_command(
"checkout",
flag,
"-q",
self.get_remote_call_options(),
rev_options.to_args(),
url,
@@ -31,12 +31,7 @@ from pip._internal.utils.misc import (
is_installable_dir,
rmtree,
)
from pip._internal.utils.subprocess import (
CommandArgs,
call_subprocess,
format_command_args,
make_command,
)
from pip._internal.utils.subprocess import CommandArgs, call_subprocess, make_command
from pip._internal.utils.urls import get_url_scheme
if TYPE_CHECKING:
@@ -463,9 +458,7 @@ class VersionControl:
"""
return cls.normalize_url(url1) == cls.normalize_url(url2)
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
"""
Fetch a revision from a repository, in the case that this is the
first fetch from the repository.
@@ -473,7 +466,6 @@ class VersionControl:
Args:
dest: the directory to fetch the repository to.
rev_options: a RevOptions object.
verbosity: verbosity level.
"""
raise NotImplementedError
@@ -506,19 +498,18 @@ class VersionControl:
"""
raise NotImplementedError
def obtain(self, dest: str, url: HiddenText, verbosity: int) -> None:
def obtain(self, dest: str, url: HiddenText) -> None:
"""
Install or update in editable mode the package represented by this
VersionControl object.
:param dest: the repository directory in which to install or update.
:param url: the repository URL starting with a vcs prefix.
:param verbosity: verbosity level.
"""
url, rev_options = self.get_url_rev_options(url)
if not os.path.exists(dest):
self.fetch_new(dest, url, rev_options, verbosity=verbosity)
self.fetch_new(dest, url, rev_options)
return
rev_display = rev_options.to_display()
@@ -574,14 +565,14 @@ class VersionControl:
if response == "w":
logger.warning("Deleting %s", display_path(dest))
rmtree(dest)
self.fetch_new(dest, url, rev_options, verbosity=verbosity)
self.fetch_new(dest, url, rev_options)
return
if response == "b":
dest_dir = backup_dir(dest)
logger.warning("Backing up %s to %s", display_path(dest), dest_dir)
shutil.move(dest, dest_dir)
self.fetch_new(dest, url, rev_options, verbosity=verbosity)
self.fetch_new(dest, url, rev_options)
return
# Do nothing if the response is "i".
@@ -595,17 +586,16 @@ class VersionControl:
)
self.switch(dest, url, rev_options)
def unpack(self, location: str, url: HiddenText, verbosity: int) -> None:
def unpack(self, location: str, url: HiddenText) -> None:
"""
Clean up current location and download the url repository
(and vcs infos) into location
:param url: the repository URL starting with a vcs prefix.
:param verbosity: verbosity level.
"""
if os.path.exists(location):
rmtree(location)
self.obtain(location, url=url, verbosity=verbosity)
self.obtain(location, url=url)
@classmethod
def get_remote_url(cls, location: str) -> str:
@@ -644,8 +634,6 @@ class VersionControl:
command name, and checks that the VCS is available
"""
cmd = make_command(cls.name, *cmd)
if command_desc is None:
command_desc = format_command_args(cmd)
try:
return call_subprocess(
cmd,
@@ -310,9 +310,7 @@ def _clean_one_legacy(req: InstallRequirement, global_options: List[str]) -> boo
logger.info("Running setup.py clean for %s", req.name)
try:
call_subprocess(
clean_args, command_desc="python setup.py clean", cwd=req.source_dir
)
call_subprocess(clean_args, cwd=req.source_dir)
return True
except Exception:
logger.error("Failed cleaning build dir for %s", req.name)