更改enroll命名,添加了注释,向get_error_msg中添加了一些错误代码

This commit is contained in:
ygm1881
2022-05-05 22:59:35 +08:00
parent 51b5e374a3
commit ece69eaf57
4637 changed files with 7699 additions and 608140 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
from typing import List, Optional
__version__ = "22.0.4"
__version__ = "21.3.1"
def main(args: Optional[List[str]] = None) -> int:
@@ -93,9 +93,7 @@ class BuildEnvironment:
self._site_dir = os.path.join(temp_dir.path, "site")
if not os.path.exists(self._site_dir):
os.mkdir(self._site_dir)
with open(
os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
) as fp:
with open(os.path.join(self._site_dir, "sitecustomize.py"), "w") as fp:
fp.write(
textwrap.dedent(
"""
@@ -189,8 +187,7 @@ class BuildEnvironment:
finder: "PackageFinder",
requirements: Iterable[str],
prefix_as_string: str,
*,
kind: str,
message: str,
) -> None:
prefix = self._prefixes[prefix_as_string]
assert not prefix.setup
@@ -198,13 +195,20 @@ class BuildEnvironment:
if not requirements:
return
with contextlib.ExitStack() as ctx:
pip_runnable = ctx.enter_context(_create_standalone_pip())
# TODO: Remove this block when dropping 3.6 support. Python 3.6
# lacks importlib.resources and pep517 has issues loading files in
# a zip, so we fallback to the "old" method by adding the current
# pip directory to the child process's sys.path.
if sys.version_info < (3, 7):
pip_runnable = os.path.dirname(pip_location)
else:
pip_runnable = ctx.enter_context(_create_standalone_pip())
self._install_requirements(
pip_runnable,
finder,
requirements,
prefix,
kind=kind,
message,
)
@staticmethod
@@ -213,8 +217,7 @@ class BuildEnvironment:
finder: "PackageFinder",
requirements: Iterable[str],
prefix: _Prefix,
*,
kind: str,
message: str,
) -> None:
args: List[str] = [
sys.executable,
@@ -256,13 +259,8 @@ class BuildEnvironment:
args.append("--")
args.extend(requirements)
extra_environ = {"_PIP_STANDALONE_CERT": where()}
with open_spinner(f"Installing {kind}") as spinner:
call_subprocess(
args,
command_desc=f"pip subprocess to install {kind}",
spinner=spinner,
extra_environ=extra_environ,
)
with open_spinner(message) as spinner:
call_subprocess(args, spinner=spinner, extra_environ=extra_environ)
class NoOpBuildEnvironment(BuildEnvironment):
@@ -290,7 +288,6 @@ class NoOpBuildEnvironment(BuildEnvironment):
finder: "PackageFinder",
requirements: Iterable[str],
prefix_as_string: str,
*,
kind: str,
message: str,
) -> None:
raise NotImplementedError()
@@ -59,14 +59,6 @@ def autocomplete() -> None:
print(dist)
sys.exit(1)
should_list_installables = (
not current.startswith("-") and subcommand_name == "install"
)
if should_list_installables:
for path in auto_complete_paths(current, "path"):
print(path)
sys.exit(1)
subcommand = create_command(subcommand_name)
for opt in subcommand.parser.option_list_all:
@@ -146,7 +138,7 @@ def auto_complete_paths(current: str, completion_type: str) -> Iterable[str]:
starting with ``current``.
:param current: The word to be completed
:param completion_type: path completion type(``file``, ``path`` or ``dir``)
:param completion_type: path completion type(`file`, `path` or `dir`)i
:return: A generator of regular files and/or directories
"""
directory, filename = os.path.split(current)
@@ -10,8 +10,6 @@ import traceback
from optparse import Values
from typing import Any, Callable, List, Optional, Tuple
from pip._vendor.rich import traceback as rich_traceback
from pip._internal.cli import cmdoptions
from pip._internal.cli.command_context import CommandContextMixIn
from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
@@ -24,7 +22,6 @@ from pip._internal.cli.status_codes import (
from pip._internal.exceptions import (
BadCommand,
CommandError,
DiagnosticPipError,
InstallationError,
NetworkConnectionError,
PreviousBuildDirError,
@@ -167,11 +164,6 @@ class Command(CommandContextMixIn):
status = run_func(*args)
assert isinstance(status, int)
return status
except DiagnosticPipError as exc:
logger.error("[present-diagnostic] %s", exc)
logger.debug("Exception information:", exc_info=True)
return ERROR
except PreviousBuildDirError as exc:
logger.critical(str(exc))
logger.debug("Exception information:", exc_info=True)
@@ -217,7 +209,6 @@ class Command(CommandContextMixIn):
run = intercepts_unhandled_exc(self.run)
else:
run = self.run
rich_traceback.install(show_locals=True)
return run(options, args)
finally:
self.handle_pip_version_check(options)
@@ -10,9 +10,9 @@ pass on state. To be consistent, all options will follow this design.
# The following comment should be removed at some point in the future.
# mypy: strict-optional=False
import logging
import os
import textwrap
import warnings
from functools import partial
from optparse import SUPPRESS_HELP, Option, OptionGroup, OptionParser, Values
from textwrap import dedent
@@ -30,8 +30,6 @@ from pip._internal.models.target_python import TargetPython
from pip._internal.utils.hashes import STRONG_HASHES
from pip._internal.utils.misc import strtobool
logger = logging.getLogger(__name__)
def raise_option_error(parser: OptionParser, option: Option, msg: str) -> None:
"""
@@ -78,9 +76,10 @@ def check_install_build_global(
if any(map(getname, names)):
control = options.format_control
control.disallow_binaries()
logger.warning(
warnings.warn(
"Disabling all use of wheels due to the use of --build-option "
"/ --global-option / --install-option.",
stacklevel=2,
)
@@ -178,15 +177,13 @@ isolated_mode: Callable[..., Option] = partial(
require_virtualenv: Callable[..., Option] = partial(
Option,
# Run only if inside a virtualenv, bail if not.
"--require-virtualenv",
"--require-venv",
dest="require_venv",
action="store_true",
default=False,
help=(
"Allow pip to only run in a virtual environment; "
"exit with an error otherwise."
),
help=SUPPRESS_HELP,
)
verbose: Callable[..., Option] = partial(
@@ -964,12 +961,7 @@ use_deprecated_feature: Callable[..., Option] = partial(
metavar="feature",
action="append",
default=[],
choices=[
"legacy-resolver",
"out-of-tree-build",
"backtrack-on-build-failures",
"html5lib",
],
choices=["legacy-resolver", "out-of-tree-build"],
help=("Enable deprecated functionality, that will be removed in the future."),
)
@@ -1,23 +1,10 @@
import functools
import itertools
import sys
from signal import SIGINT, default_int_handler, signal
from typing import Any, Callable, Iterator, Optional, Tuple
from typing import Any
from pip._vendor.progress.bar import Bar, FillingCirclesBar, IncrementalBar
from pip._vendor.progress.spinner import Spinner
from pip._vendor.rich.progress import (
BarColumn,
DownloadColumn,
FileSizeColumn,
Progress,
ProgressColumn,
SpinnerColumn,
TextColumn,
TimeElapsedColumn,
TimeRemainingColumn,
TransferSpeedColumn,
)
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.logging import get_indentation
@@ -30,8 +17,6 @@ try:
except Exception:
colorama = None
DownloadProgressRenderer = Callable[[Iterator[bytes]], Iterator[bytes]]
def _select_progress_class(preferred: Bar, fallback: Bar) -> Bar:
encoding = getattr(preferred.file, "encoding", None)
@@ -258,64 +243,8 @@ BAR_TYPES = {
}
def _legacy_progress_bar(
progress_bar: str, max: Optional[int]
) -> DownloadProgressRenderer:
def DownloadProgressProvider(progress_bar, max=None): # type: ignore
if max is None or max == 0:
return BAR_TYPES[progress_bar][1]().iter # type: ignore
return BAR_TYPES[progress_bar][1]().iter
else:
return BAR_TYPES[progress_bar][0](max=max).iter
#
# Modern replacement, for our legacy progress bars.
#
def _rich_progress_bar(
iterable: Iterator[bytes],
*,
bar_type: str,
size: int,
) -> Iterator[bytes]:
assert bar_type == "on", "This should only be used in the default mode."
if not size:
total = float("inf")
columns: Tuple[ProgressColumn, ...] = (
TextColumn("[progress.description]{task.description}"),
SpinnerColumn("line", speed=1.5),
FileSizeColumn(),
TransferSpeedColumn(),
TimeElapsedColumn(),
)
else:
total = size
columns = (
TextColumn("[progress.description]{task.description}"),
BarColumn(),
DownloadColumn(),
TransferSpeedColumn(),
TextColumn("eta"),
TimeRemainingColumn(),
)
progress = Progress(*columns, refresh_per_second=30)
task_id = progress.add_task(" " * (get_indentation() + 2), total=total)
with progress:
for chunk in iterable:
yield chunk
progress.update(task_id, advance=len(chunk))
def get_download_progress_renderer(
*, bar_type: str, size: Optional[int] = None
) -> DownloadProgressRenderer:
"""Get an object that can be used to render the download progress.
Returns a callable, that takes an iterable to "wrap".
"""
if bar_type == "on":
return functools.partial(_rich_progress_bar, bar_type=bar_type, size=size)
elif bar_type == "off":
return iter # no-op, when passed an iterator
else:
return _legacy_progress_bar(bar_type, size)
@@ -227,31 +227,6 @@ class RequirementCommand(IndexGroupCommand):
return "2020-resolver"
@staticmethod
def determine_build_failure_suppression(options: Values) -> bool:
"""Determines whether build failures should be suppressed and backtracked on."""
if "backtrack-on-build-failures" not in options.deprecated_features_enabled:
return False
if "legacy-resolver" in options.deprecated_features_enabled:
raise CommandError("Cannot backtrack with legacy resolver.")
deprecated(
reason=(
"Backtracking on build failures can mask issues related to how "
"a package generates metadata or builds a wheel. This flag will "
"be removed in pip 22.2."
),
gone_in=None,
replacement=(
"avoiding known-bad versions by explicitly telling pip to ignore them "
"(either directly as requirements, or via a constraints file)"
),
feature_flag=None,
issue=10655,
)
return True
@classmethod
def make_requirement_preparer(
cls,
@@ -262,7 +237,6 @@ class RequirementCommand(IndexGroupCommand):
finder: PackageFinder,
use_user_site: bool,
download_dir: Optional[str] = None,
verbosity: int = 0,
) -> RequirementPreparer:
"""
Create a RequirementPreparer instance for the given parameters.
@@ -302,13 +276,6 @@ class RequirementCommand(IndexGroupCommand):
gone_in="22.1",
)
if options.progress_bar not in {"on", "off"}:
deprecated(
reason="Custom progress bar styles are deprecated",
replacement="to use the default progress bar style.",
gone_in="22.1",
)
return RequirementPreparer(
build_dir=temp_build_dir_path,
src_dir=options.src_dir,
@@ -321,7 +288,6 @@ class RequirementCommand(IndexGroupCommand):
require_hashes=options.require_hashes,
use_user_site=use_user_site,
lazy_wheel=lazy_wheel,
verbosity=verbosity,
in_tree_build=in_tree_build,
)
@@ -348,7 +314,6 @@ class RequirementCommand(IndexGroupCommand):
isolated=options.isolated_mode,
use_pep517=use_pep517,
)
suppress_build_failures = cls.determine_build_failure_suppression(options)
resolver_variant = cls.determine_resolver_variant(options)
# The long import name and duplicated invocation is needed to convince
# Mypy into correctly typechecking. Otherwise it would complain the
@@ -368,7 +333,6 @@ class RequirementCommand(IndexGroupCommand):
force_reinstall=force_reinstall,
upgrade_strategy=upgrade_strategy,
py_version_info=py_version_info,
suppress_build_failures=suppress_build_failures,
)
import pip._internal.resolution.legacy.resolver
@@ -502,5 +466,4 @@ class RequirementCommand(IndexGroupCommand):
link_collector=link_collector,
selection_prefs=selection_prefs,
target_python=target_python,
use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
)
@@ -113,7 +113,6 @@ class DownloadCommand(RequirementCommand):
finder=finder,
download_dir=options.download_dir,
use_user_site=False,
verbosity=self.verbosity,
)
resolver = self.make_resolver(
@@ -97,7 +97,6 @@ class IndexCommand(IndexGroupCommand):
link_collector=link_collector,
selection_prefs=selection_prefs,
target_python=target_python,
use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
)
def get_available_package_versions(self, options: Values, args: List[Any]) -> None:
@@ -319,7 +319,6 @@ class InstallCommand(RequirementCommand):
session=session,
finder=finder,
use_user_site=options.use_user_site,
verbosity=self.verbosity,
)
resolver = self.make_resolver(
preparer=preparer,
@@ -16,6 +16,7 @@ from pip._internal.models.selection_prefs import SelectionPreferences
from pip._internal.network.session import PipSession
from pip._internal.utils.compat import stdlib_pkgs
from pip._internal.utils.misc import tabulate, write_output
from pip._internal.utils.parallel import map_multithread
if TYPE_CHECKING:
from pip._internal.metadata.base import DistributionVersion
@@ -149,7 +150,6 @@ class ListCommand(IndexGroupCommand):
return PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
)
def run(self, options: Values, args: List[str]) -> int:
@@ -254,7 +254,7 @@ class ListCommand(IndexGroupCommand):
dist.latest_filetype = typ
return dist
for dist in map(latest_info, packages):
for dist in map_multithread(latest_info, packages):
if dist is not None:
yield dist
@@ -1,6 +1,8 @@
import csv
import logging
import pathlib
from optparse import Values
from typing import Iterator, List, NamedTuple, Optional
from typing import Iterator, List, NamedTuple, Optional, Tuple
from pip._vendor.packaging.utils import canonicalize_name
@@ -67,6 +69,33 @@ class _PackageInfo(NamedTuple):
files: Optional[List[str]]
def _convert_legacy_entry(entry: Tuple[str, ...], info: Tuple[str, ...]) -> str:
"""Convert a legacy installed-files.txt path into modern RECORD path.
The legacy format stores paths relative to the info directory, while the
modern format stores paths relative to the package root, e.g. the
site-packages directory.
:param entry: Path parts of the installed-files.txt entry.
:param info: Path parts of the egg-info directory relative to package root.
:returns: The converted entry.
For best compatibility with symlinks, this does not use ``abspath()`` or
``Path.resolve()``, but tries to work with path parts:
1. While ``entry`` starts with ``..``, remove the equal amounts of parts
from ``info``; if ``info`` is empty, start appending ``..`` instead.
2. Join the two directly.
"""
while entry and entry[0] == "..":
if not info or info[-1] == "..":
info += ("..",)
else:
info = info[:-1]
entry = entry[1:]
return str(pathlib.Path(*info, *entry))
def search_packages_info(query: List[str]) -> Iterator[_PackageInfo]:
"""
Gather details from installed distributions. Print distribution name,
@@ -92,6 +121,34 @@ def search_packages_info(query: List[str]) -> Iterator[_PackageInfo]:
in {canonicalize_name(d.name) for d in dist.iter_dependencies()}
)
def _files_from_record(dist: BaseDistribution) -> Optional[Iterator[str]]:
try:
text = dist.read_text("RECORD")
except FileNotFoundError:
return None
# This extra Path-str cast normalizes entries.
return (str(pathlib.Path(row[0])) for row in csv.reader(text.splitlines()))
def _files_from_legacy(dist: BaseDistribution) -> Optional[Iterator[str]]:
try:
text = dist.read_text("installed-files.txt")
except FileNotFoundError:
return None
paths = (p for p in text.splitlines(keepends=False) if p)
root = dist.location
info = dist.info_directory
if root is None or info is None:
return paths
try:
info_rel = pathlib.Path(info).relative_to(root)
except ValueError: # info is not relative to root.
return paths
if not info_rel.parts: # info *is* root.
return paths
return (
_convert_legacy_entry(pathlib.Path(p).parts, info_rel.parts) for p in paths
)
for query_name in query_names:
try:
dist = installed[query_name]
@@ -107,7 +164,7 @@ def search_packages_info(query: List[str]) -> Iterator[_PackageInfo]:
except FileNotFoundError:
entry_points = []
files_iter = dist.iter_declared_entries()
files_iter = _files_from_record(dist) or _files_from_legacy(dist)
if files_iter is None:
files: Optional[List[str]] = None
else:
@@ -128,7 +128,6 @@ class WheelCommand(RequirementCommand):
finder=finder,
download_dir=options.wheel_dir,
use_user_site=False,
verbosity=self.verbosity,
)
resolver = self.make_resolver(
@@ -266,13 +266,14 @@ class Configuration:
# Doing this is useful when modifying and saving files, where we don't
# need to construct a parser.
if os.path.exists(fname):
locale_encoding = locale.getpreferredencoding(False)
try:
parser.read(fname, encoding=locale_encoding)
parser.read(fname)
except UnicodeDecodeError:
# See https://github.com/pypa/pip/issues/4963
raise ConfigurationFileCouldNotBeLoaded(
reason=f"contains invalid {locale_encoding} characters",
reason="contains invalid {} characters".format(
locale.getpreferredencoding(False)
),
fname=fname,
)
except configparser.Error as error:
@@ -11,8 +11,10 @@ class InstalledDistribution(AbstractDistribution):
"""
def get_metadata_distribution(self) -> BaseDistribution:
from pip._internal.metadata.pkg_resources import Distribution as _Dist
assert self.req.satisfied_by is not None, "not actually installed"
return self.req.satisfied_by
return _Dist(self.req.satisfied_by)
def prepare_distribution_metadata(
self, finder: PackageFinder, build_isolation: bool
@@ -19,7 +19,9 @@ class SourceDistribution(AbstractDistribution):
"""
def get_metadata_distribution(self) -> BaseDistribution:
return self.req.get_dist()
from pip._internal.metadata.pkg_resources import Distribution as _Dist
return _Dist(self.req.get_dist())
def prepare_distribution_metadata(
self, finder: PackageFinder, build_isolation: bool
@@ -54,7 +56,7 @@ class SourceDistribution(AbstractDistribution):
self.req.build_env = BuildEnvironment()
self.req.build_env.install_requirements(
finder, pyproject_requires, "overlay", kind="build dependencies"
finder, pyproject_requires, "overlay", "Installing build dependencies"
)
conflicting, missing = self.req.build_env.check_requirements(
self.req.requirements_to_check
@@ -106,7 +108,7 @@ class SourceDistribution(AbstractDistribution):
if conflicting:
self._raise_conflicts("the backend dependencies", conflicting)
self.req.build_env.install_requirements(
finder, missing, "normal", kind="backend dependencies"
finder, missing, "normal", "Installing backend dependencies"
)
def _raise_conflicts(
@@ -1,174 +1,23 @@
"""Exceptions used throughout package.
This module MUST NOT try to import from anything within `pip._internal` to
operate. This is expected to be importable from any/all files within the
subpackage and, thus, should not depend on them.
"""
"""Exceptions used throughout package"""
import configparser
import re
from itertools import chain, groupby, repeat
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from pip._vendor.pkg_resources import Distribution
from pip._vendor.requests.models import Request, Response
from pip._vendor.rich.console import Console, ConsoleOptions, RenderResult
from pip._vendor.rich.markup import escape
from pip._vendor.rich.text import Text
if TYPE_CHECKING:
from hashlib import _Hash
from typing import Literal
from pip._internal.metadata import BaseDistribution
from pip._internal.req.req_install import InstallRequirement
#
# Scaffolding
#
def _is_kebab_case(s: str) -> bool:
return re.match(r"^[a-z]+(-[a-z]+)*$", s) is not None
def _prefix_with_indent(
s: Union[Text, str],
console: Console,
*,
prefix: str,
indent: str,
) -> Text:
if isinstance(s, Text):
text = s
else:
text = console.render_str(s)
return console.render_str(prefix, overflow="ignore") + console.render_str(
f"\n{indent}", overflow="ignore"
).join(text.split(allow_blank=True))
class PipError(Exception):
"""The base pip error."""
"""Base pip exception"""
class DiagnosticPipError(PipError):
"""An error, that presents diagnostic information to the user.
This contains a bunch of logic, to enable pretty presentation of our error
messages. Each error gets a unique reference. Each error can also include
additional context, a hint and/or a note -- which are presented with the
main error message in a consistent style.
This is adapted from the error output styling in `sphinx-theme-builder`.
"""
reference: str
def __init__(
self,
*,
kind: 'Literal["error", "warning"]' = "error",
reference: Optional[str] = None,
message: Union[str, Text],
context: Optional[Union[str, Text]],
hint_stmt: Optional[Union[str, Text]],
note_stmt: Optional[Union[str, Text]] = None,
link: Optional[str] = None,
) -> None:
# Ensure a proper reference is provided.
if reference is None:
assert hasattr(self, "reference"), "error reference not provided!"
reference = self.reference
assert _is_kebab_case(reference), "error reference must be kebab-case!"
self.kind = kind
self.reference = reference
self.message = message
self.context = context
self.note_stmt = note_stmt
self.hint_stmt = hint_stmt
self.link = link
super().__init__(f"<{self.__class__.__name__}: {self.reference}>")
def __repr__(self) -> str:
return (
f"<{self.__class__.__name__}("
f"reference={self.reference!r}, "
f"message={self.message!r}, "
f"context={self.context!r}, "
f"note_stmt={self.note_stmt!r}, "
f"hint_stmt={self.hint_stmt!r}"
")>"
)
def __rich_console__(
self,
console: Console,
options: ConsoleOptions,
) -> RenderResult:
colour = "red" if self.kind == "error" else "yellow"
yield f"[{colour} bold]{self.kind}[/]: [bold]{self.reference}[/]"
yield ""
if not options.ascii_only:
# Present the main message, with relevant context indented.
if self.context is not None:
yield _prefix_with_indent(
self.message,
console,
prefix=f"[{colour}]×[/] ",
indent=f"[{colour}]│[/] ",
)
yield _prefix_with_indent(
self.context,
console,
prefix=f"[{colour}]╰─>[/] ",
indent=f"[{colour}] [/] ",
)
else:
yield _prefix_with_indent(
self.message,
console,
prefix="[red]×[/] ",
indent=" ",
)
else:
yield self.message
if self.context is not None:
yield ""
yield self.context
if self.note_stmt is not None or self.hint_stmt is not None:
yield ""
if self.note_stmt is not None:
yield _prefix_with_indent(
self.note_stmt,
console,
prefix="[magenta bold]note[/]: ",
indent=" ",
)
if self.hint_stmt is not None:
yield _prefix_with_indent(
self.hint_stmt,
console,
prefix="[cyan bold]hint[/]: ",
indent=" ",
)
if self.link is not None:
yield ""
yield f"Link: {self.link}"
#
# Actual Errors
#
class ConfigurationError(PipError):
"""General exception in configuration"""
@@ -181,52 +30,18 @@ class UninstallationError(PipError):
"""General exception during uninstallation"""
class MissingPyProjectBuildRequires(DiagnosticPipError):
"""Raised when pyproject.toml has `build-system`, but no `build-system.requires`."""
reference = "missing-pyproject-build-system-requires"
def __init__(self, *, package: str) -> None:
super().__init__(
message=f"Can not process {escape(package)}",
context=Text(
"This package has an invalid pyproject.toml file.\n"
"The [build-system] table is missing the mandatory `requires` key."
),
note_stmt="This is an issue with the package mentioned above, not pip.",
hint_stmt=Text("See PEP 518 for the detailed specification."),
)
class InvalidPyProjectBuildRequires(DiagnosticPipError):
"""Raised when pyproject.toml an invalid `build-system.requires`."""
reference = "invalid-pyproject-build-system-requires"
def __init__(self, *, package: str, reason: str) -> None:
super().__init__(
message=f"Can not process {escape(package)}",
context=Text(
"This package has an invalid `build-system.requires` key in "
f"pyproject.toml.\n{reason}"
),
note_stmt="This is an issue with the package mentioned above, not pip.",
hint_stmt=Text("See PEP 518 for the detailed specification."),
)
class NoneMetadataError(PipError):
"""Raised when accessing a Distribution's "METADATA" or "PKG-INFO".
This signifies an inconsistency, when the Distribution claims to have
the metadata file (if not, raise ``FileNotFoundError`` instead), but is
not actually able to produce its content. This may be due to permission
errors.
"""
Raised when accessing "METADATA" or "PKG-INFO" metadata for a
pip._vendor.pkg_resources.Distribution object and
`dist.has_metadata('METADATA')` returns True but
`dist.get_metadata('METADATA')` returns None (and similarly for
"PKG-INFO").
"""
def __init__(
self,
dist: "BaseDistribution",
dist: Union[Distribution, "BaseDistribution"],
metadata_name: str,
) -> None:
"""
@@ -317,17 +132,6 @@ class UnsupportedWheel(InstallationError):
"""Unsupported wheel."""
class InvalidWheel(InstallationError):
"""Invalid (e.g. corrupt) wheel."""
def __init__(self, location: str, name: str):
self.location = location
self.name = name
def __str__(self) -> str:
return f"Wheel '{self.name}' located at {self.location} is invalid."
class MetadataInconsistent(InstallationError):
"""Built metadata contains inconsistent information.
@@ -352,78 +156,18 @@ class MetadataInconsistent(InstallationError):
return template.format(self.ireq, self.field, self.f_val, self.m_val)
class LegacyInstallFailure(DiagnosticPipError):
"""Error occurred while executing `setup.py install`"""
class InstallationSubprocessError(InstallationError):
"""A subprocess call failed during installation."""
reference = "legacy-install-failure"
def __init__(self, package_details: str) -> None:
super().__init__(
message="Encountered error while trying to install package.",
context=package_details,
hint_stmt="See above for output from the failure.",
note_stmt="This is an issue with the package mentioned above, not pip.",
)
class InstallationSubprocessError(DiagnosticPipError, InstallationError):
"""A subprocess call failed."""
reference = "subprocess-exited-with-error"
def __init__(
self,
*,
command_description: str,
exit_code: int,
output_lines: Optional[List[str]],
) -> None:
if output_lines is None:
output_prompt = Text("See above for output.")
else:
output_prompt = (
Text.from_markup(f"[red][{len(output_lines)} lines of output][/]\n")
+ Text("".join(output_lines))
+ Text.from_markup(R"[red]\[end of output][/]")
)
super().__init__(
message=(
f"[green]{escape(command_description)}[/] did not run successfully.\n"
f"exit code: {exit_code}"
),
context=output_prompt,
hint_stmt=None,
note_stmt=(
"This error originates from a subprocess, and is likely not a "
"problem with pip."
),
)
self.command_description = command_description
self.exit_code = exit_code
def __init__(self, returncode: int, description: str) -> None:
self.returncode = returncode
self.description = description
def __str__(self) -> str:
return f"{self.command_description} exited with {self.exit_code}"
class MetadataGenerationFailed(InstallationSubprocessError, InstallationError):
reference = "metadata-generation-failed"
def __init__(
self,
*,
package_details: str,
) -> None:
super(InstallationSubprocessError, self).__init__(
message="Encountered error while generating package metadata.",
context=escape(package_details),
hint_stmt="See above for details.",
note_stmt="This is an issue with the package mentioned above, not pip.",
)
def __str__(self) -> str:
return "metadata generation failed"
return (
"Command errored out with exit status {}: {} "
"Check the logs for full command output."
).format(self.returncode, self.description)
class HashErrors(InstallationError):
@@ -12,19 +12,15 @@ import re
import urllib.parse
import urllib.request
import xml.etree.ElementTree
from html.parser import HTMLParser
from optparse import Values
from typing import (
TYPE_CHECKING,
Callable,
Dict,
Iterable,
List,
MutableMapping,
NamedTuple,
Optional,
Sequence,
Tuple,
Union,
)
@@ -43,11 +39,6 @@ from pip._internal.vcs import vcs
from .sources import CandidatesFromPage, LinkSource, build_source
if TYPE_CHECKING:
from typing import Protocol
else:
Protocol = object
logger = logging.getLogger(__name__)
HTMLElement = xml.etree.ElementTree.Element
@@ -172,8 +163,6 @@ def _determine_base_url(document: HTMLElement, page_url: str) -> str:
:param document: An HTML document representation. The current
implementation expects the result of ``html5lib.parse()``.
:param page_url: The URL of the HTML document.
TODO: Remove when `html5lib` is dropped.
"""
for base in document.findall(".//base"):
href = base.get("href")
@@ -245,20 +234,20 @@ def _clean_link(url: str) -> str:
def _create_link_from_element(
element_attribs: Dict[str, Optional[str]],
anchor: HTMLElement,
page_url: str,
base_url: str,
) -> Optional[Link]:
"""
Convert an anchor element's attributes in a simple repository page to a Link.
Convert an anchor element in a simple repository page to a Link.
"""
href = element_attribs.get("href")
href = anchor.get("href")
if not href:
return None
url = _clean_link(urllib.parse.urljoin(base_url, href))
pyrequire = element_attribs.get("data-requires-python")
yanked_reason = element_attribs.get("data-yanked")
pyrequire = anchor.get("data-requires-python")
yanked_reason = anchor.get("data-yanked")
link = Link(
url,
@@ -282,14 +271,9 @@ class CacheablePageContent:
return hash(self.page.url)
class ParseLinks(Protocol):
def __call__(
self, page: "HTMLPage", use_deprecated_html5lib: bool
) -> Iterable[Link]:
...
def with_cached_html_pages(fn: ParseLinks) -> ParseLinks:
def with_cached_html_pages(
fn: Callable[["HTMLPage"], Iterable[Link]],
) -> Callable[["HTMLPage"], List[Link]]:
"""
Given a function that parses an Iterable[Link] from an HTMLPage, cache the
function's result (keyed by CacheablePageContent), unless the HTMLPage
@@ -297,25 +281,22 @@ def with_cached_html_pages(fn: ParseLinks) -> ParseLinks:
"""
@functools.lru_cache(maxsize=None)
def wrapper(
cacheable_page: CacheablePageContent, use_deprecated_html5lib: bool
) -> List[Link]:
return list(fn(cacheable_page.page, use_deprecated_html5lib))
def wrapper(cacheable_page: CacheablePageContent) -> List[Link]:
return list(fn(cacheable_page.page))
@functools.wraps(fn)
def wrapper_wrapper(page: "HTMLPage", use_deprecated_html5lib: bool) -> List[Link]:
def wrapper_wrapper(page: "HTMLPage") -> List[Link]:
if page.cache_link_parsing:
return wrapper(CacheablePageContent(page), use_deprecated_html5lib)
return list(fn(page, use_deprecated_html5lib))
return wrapper(CacheablePageContent(page))
return list(fn(page))
return wrapper_wrapper
def _parse_links_html5lib(page: "HTMLPage") -> Iterable[Link]:
@with_cached_html_pages
def parse_links(page: "HTMLPage") -> Iterable[Link]:
"""
Parse an HTML document, and yield its anchor elements as Link objects.
TODO: Remove when `html5lib` is dropped.
"""
document = html5lib.parse(
page.content,
@@ -326,33 +307,6 @@ def _parse_links_html5lib(page: "HTMLPage") -> Iterable[Link]:
url = page.url
base_url = _determine_base_url(document, url)
for anchor in document.findall(".//a"):
link = _create_link_from_element(
anchor.attrib,
page_url=url,
base_url=base_url,
)
if link is None:
continue
yield link
@with_cached_html_pages
def parse_links(page: "HTMLPage", use_deprecated_html5lib: bool) -> Iterable[Link]:
"""
Parse an HTML document, and yield its anchor elements as Link objects.
"""
if use_deprecated_html5lib:
yield from _parse_links_html5lib(page)
return
parser = HTMLLinkParser(page.url)
encoding = page.encoding or "utf-8"
parser.feed(page.content.decode(encoding))
url = page.url
base_url = parser.base_url or url
for anchor in parser.anchors:
link = _create_link_from_element(
anchor,
page_url=url,
@@ -389,34 +343,6 @@ class HTMLPage:
return redact_auth_from_url(self.url)
class HTMLLinkParser(HTMLParser):
"""
HTMLParser that keeps the first base HREF and a list of all anchor
elements' attributes.
"""
def __init__(self, url: str) -> None:
super().__init__(convert_charrefs=True)
self.url: str = url
self.base_url: Optional[str] = None
self.anchors: List[Dict[str, Optional[str]]] = []
def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
if tag == "base" and self.base_url is None:
href = self.get_href(attrs)
if href is not None:
self.base_url = href
elif tag == "a":
self.anchors.append(dict(attrs))
def get_href(self, attrs: List[Tuple[str, Optional[str]]]) -> Optional[str]:
for name, value in attrs:
if name == "href":
return value
return None
def _handle_get_page_fail(
link: Link,
reason: Union[str, Exception],
@@ -37,6 +37,7 @@ from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import build_netloc
from pip._internal.utils.packaging import check_requires_python
from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS
from pip._internal.utils.urls import url_to_path
__all__ = ["FormatControl", "BestCandidateResult", "PackageFinder"]
@@ -580,7 +581,6 @@ class PackageFinder:
link_collector: LinkCollector,
target_python: TargetPython,
allow_yanked: bool,
use_deprecated_html5lib: bool,
format_control: Optional[FormatControl] = None,
candidate_prefs: Optional[CandidatePreferences] = None,
ignore_requires_python: Optional[bool] = None,
@@ -605,7 +605,6 @@ class PackageFinder:
self._ignore_requires_python = ignore_requires_python
self._link_collector = link_collector
self._target_python = target_python
self._use_deprecated_html5lib = use_deprecated_html5lib
self.format_control = format_control
@@ -622,8 +621,6 @@ class PackageFinder:
link_collector: LinkCollector,
selection_prefs: SelectionPreferences,
target_python: Optional[TargetPython] = None,
*,
use_deprecated_html5lib: bool,
) -> "PackageFinder":
"""Create a PackageFinder.
@@ -648,7 +645,6 @@ class PackageFinder:
allow_yanked=selection_prefs.allow_yanked,
format_control=selection_prefs.format_control,
ignore_requires_python=selection_prefs.ignore_requires_python,
use_deprecated_html5lib=use_deprecated_html5lib,
)
@property
@@ -770,7 +766,7 @@ class PackageFinder:
if html_page is None:
return []
page_links = list(parse_links(html_page, self._use_deprecated_html5lib))
page_links = list(parse_links(html_page))
with indent_log():
package_links = self.evaluate_links(
@@ -820,14 +816,7 @@ class PackageFinder:
)
if logger.isEnabledFor(logging.DEBUG) and file_candidates:
paths = []
for candidate in file_candidates:
assert candidate.link.url # we need to have a URL
try:
paths.append(candidate.link.file_path)
except Exception:
paths.append(candidate.link.url) # it's not a local file
paths = [url_to_path(c.link.url) for c in file_candidates]
logger.debug("Local files found: %s", ", ".join(paths))
# This is an intentional priority ordering
@@ -892,7 +881,7 @@ class PackageFinder:
installed_version: Optional[_BaseVersion] = None
if req.satisfied_by is not None:
installed_version = req.satisfied_by.version
installed_version = parse_version(req.satisfied_by.version)
def _format_versions(cand_iter: Iterable[InstallationCandidate]) -> str:
# This repeated parse_version and str() conversion is needed to
@@ -38,34 +38,14 @@ __all__ = [
logger = logging.getLogger(__name__)
if os.environ.get("_PIP_LOCATIONS_NO_WARN_ON_MISMATCH"):
_MISMATCH_LEVEL = logging.DEBUG
else:
_MISMATCH_LEVEL = logging.WARNING
_PLATLIBDIR: str = getattr(sys, "platlibdir", "lib")
_USE_SYSCONFIG_DEFAULT = sys.version_info >= (3, 10)
def _should_use_sysconfig() -> bool:
"""This function determines the value of _USE_SYSCONFIG.
By default, pip uses sysconfig on Python 3.10+.
But Python distributors can override this decision by setting:
sysconfig._PIP_USE_SYSCONFIG = True / False
Rationale in https://github.com/pypa/pip/issues/10647
This is a function for testability, but should be constant during any one
run.
"""
return bool(getattr(sysconfig, "_PIP_USE_SYSCONFIG", _USE_SYSCONFIG_DEFAULT))
_USE_SYSCONFIG = _should_use_sysconfig()
# Be noisy about incompatibilities if this platforms "should" be using
# sysconfig, but is explicitly opting out and using distutils instead.
if _USE_SYSCONFIG_DEFAULT and not _USE_SYSCONFIG:
_MISMATCH_LEVEL = logging.WARNING
else:
_MISMATCH_LEVEL = logging.DEBUG
_USE_SYSCONFIG = sys.version_info >= (3, 10)
def _looks_like_bpo_44860() -> bool:
@@ -84,8 +64,8 @@ def _looks_like_bpo_44860() -> bool:
def _looks_like_red_hat_patched_platlib_purelib(scheme: Dict[str, str]) -> bool:
platlib = scheme["platlib"]
if "/$platlibdir/" in platlib:
platlib = platlib.replace("/$platlibdir/", f"/{_PLATLIBDIR}/")
if "/$platlibdir/" in platlib and hasattr(sys, "platlibdir"):
platlib = platlib.replace("/$platlibdir/", f"/{sys.platlibdir}/")
if "/lib64/" not in platlib:
return False
unpatched = platlib.replace("/lib64/", "/lib/")
@@ -135,22 +115,6 @@ def _looks_like_red_hat_scheme() -> bool:
)
@functools.lru_cache(maxsize=None)
def _looks_like_slackware_scheme() -> bool:
"""Slackware patches sysconfig but fails to patch distutils and site.
Slackware changes sysconfig's user scheme to use ``"lib64"`` for the lib
path, but does not do the same to the site module.
"""
if user_site is None: # User-site not available.
return False
try:
paths = sysconfig.get_paths(scheme="posix_user", expand=False)
except KeyError: # User-site not available.
return False
return "/lib64/" in paths["purelib"] and "/lib64/" not in user_site
@functools.lru_cache(maxsize=None)
def _looks_like_msys2_mingw_scheme() -> bool:
"""MSYS2 patches distutils and sysconfig to use a UNIX-like scheme.
@@ -306,17 +270,6 @@ def get_scheme(
if skip_bpo_44860:
continue
# Slackware incorrectly patches posix_user to use lib64 instead of lib,
# but not usersite to match the location.
skip_slackware_user_scheme = (
user
and k in ("platlib", "purelib")
and not WINDOWS
and _looks_like_slackware_scheme()
)
if skip_slackware_user_scheme:
continue
# Both Debian and Red Hat patch Python to place the system site under
# /usr/local instead of /usr. Debian also places lib in dist-packages
# instead of site-packages, but the /usr/local check should cover it.
@@ -467,13 +420,6 @@ def _deduplicated(v1: str, v2: str) -> List[str]:
return [v1, v2]
def _looks_like_apple_library(path: str) -> bool:
"""Apple patches sysconfig to *always* look under */Library/Python*."""
if sys.platform[:6] != "darwin":
return False
return path == f"/Library/Python/{get_major_minor_version()}/site-packages"
def get_prefixed_libs(prefix: str) -> List[str]:
"""Return the lib locations under ``prefix``."""
new_pure, new_plat = _sysconfig.get_prefixed_libs(prefix)
@@ -481,26 +427,6 @@ def get_prefixed_libs(prefix: str) -> List[str]:
return _deduplicated(new_pure, new_plat)
old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
old_lib_paths = _deduplicated(old_pure, old_plat)
# Apple's Python (shipped with Xcode and Command Line Tools) hard-code
# platlib and purelib to '/Library/Python/X.Y/site-packages'. This will
# cause serious build isolation bugs when Apple starts shipping 3.10 because
# pip will install build backends to the wrong location. This tells users
# who is at fault so Apple may notice it and fix the issue in time.
if all(_looks_like_apple_library(p) for p in old_lib_paths):
deprecated(
reason=(
"Python distributed by Apple's Command Line Tools incorrectly "
"patches sysconfig to always point to '/Library/Python'. This "
"will cause build isolation to operate incorrectly on Python "
"3.10 or later. Please help report this to Apple so they can "
"fix this. https://developer.apple.com/bug-reporting/"
),
replacement=None,
gone_in=None,
)
return old_lib_paths
warned = [
_warn_if_mismatch(
@@ -517,4 +443,4 @@ def get_prefixed_libs(prefix: str) -> List[str]:
if any(warned):
_log_context(prefix=prefix)
return old_lib_paths
return _deduplicated(old_pure, old_plat)
@@ -38,17 +38,6 @@ def get_environment(paths: Optional[List[str]]) -> BaseEnvironment:
return Environment.from_paths(paths)
def get_directory_distribution(directory: str) -> BaseDistribution:
"""Get the distribution metadata representation in the specified directory.
This returns a Distribution instance from the chosen backend based on
the given on-disk ``.dist-info`` directory.
"""
from .pkg_resources import Distribution
return Distribution.from_directory(directory)
def get_wheel_distribution(wheel: Wheel, canonical_name: str) -> BaseDistribution:
"""Get the representation of the specified wheel's distribution metadata.
@@ -1,8 +1,6 @@
import csv
import email.message
import json
import logging
import pathlib
import re
import zipfile
from typing import (
@@ -14,7 +12,6 @@ from typing import (
Iterator,
List,
Optional,
Tuple,
Union,
)
@@ -23,19 +20,13 @@ from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
from pip._vendor.packaging.utils import NormalizedName
from pip._vendor.packaging.version import LegacyVersion, Version
from pip._internal.exceptions import NoneMetadataError
from pip._internal.locations import site_packages, user_site
from pip._internal.models.direct_url import (
DIRECT_URL_METADATA_NAME,
DirectUrl,
DirectUrlValidationError,
)
from pip._internal.utils.compat import stdlib_pkgs # TODO: Move definition here.
from pip._internal.utils.egg_link import (
egg_link_path_from_location,
egg_link_path_from_sys_path,
)
from pip._internal.utils.misc import is_local, normalize_path
from pip._internal.utils.egg_link import egg_link_path_from_sys_path
from pip._internal.utils.urls import url_to_path
if TYPE_CHECKING:
@@ -45,8 +36,6 @@ else:
DistributionVersion = Union[LegacyVersion, Version]
InfoPath = Union[str, pathlib.PurePosixPath]
logger = logging.getLogger(__name__)
@@ -64,36 +53,6 @@ class BaseEntryPoint(Protocol):
raise NotImplementedError()
def _convert_installed_files_path(
entry: Tuple[str, ...],
info: Tuple[str, ...],
) -> str:
"""Convert a legacy installed-files.txt path into modern RECORD path.
The legacy format stores paths relative to the info directory, while the
modern format stores paths relative to the package root, e.g. the
site-packages directory.
:param entry: Path parts of the installed-files.txt entry.
:param info: Path parts of the egg-info directory relative to package root.
:returns: The converted entry.
For best compatibility with symlinks, this does not use ``abspath()`` or
``Path.resolve()``, but tries to work with path parts:
1. While ``entry`` starts with ``..``, remove the equal amounts of parts
from ``info``; if ``info`` is empty, start appending ``..`` instead.
2. Join the two directly.
"""
while entry and entry[0] == "..":
if not info or info[-1] == "..":
info += ("..",)
else:
info = info[:-1]
entry = entry[1:]
return str(pathlib.Path(*info, *entry))
class BaseDistribution(Protocol):
def __repr__(self) -> str:
return f"{self.raw_name} {self.version} ({self.location})"
@@ -138,28 +97,8 @@ class BaseDistribution(Protocol):
return None
@property
def installed_location(self) -> Optional[str]:
"""The distribution's "installed" location.
This should generally be a ``site-packages`` directory. This is
usually ``dist.location``, except for legacy develop-installed packages,
where ``dist.location`` is the source code location, and this is where
the ``.egg-link`` file is.
The returned location is normalized (in particular, with symlinks removed).
"""
egg_link = egg_link_path_from_location(self.raw_name)
if egg_link:
location = egg_link
elif self.location:
location = self.location
else:
return None
return normalize_path(location)
@property
def info_location(self) -> Optional[str]:
"""Location of the .[egg|dist]-info directory or file.
def info_directory(self) -> Optional[str]:
"""Location of the .[egg|dist]-info directory.
Similarly to ``location``, a string value is not necessarily a
filesystem path. ``None`` means the distribution is created in-memory.
@@ -173,65 +112,6 @@ class BaseDistribution(Protocol):
"""
raise NotImplementedError()
@property
def installed_by_distutils(self) -> bool:
"""Whether this distribution is installed with legacy distutils format.
A distribution installed with "raw" distutils not patched by setuptools
uses one single file at ``info_location`` to store metadata. We need to
treat this specially on uninstallation.
"""
info_location = self.info_location
if not info_location:
return False
return pathlib.Path(info_location).is_file()
@property
def installed_as_egg(self) -> bool:
"""Whether this distribution is installed as an egg.
This usually indicates the distribution was installed by (older versions
of) easy_install.
"""
location = self.location
if not location:
return False
return location.endswith(".egg")
@property
def installed_with_setuptools_egg_info(self) -> bool:
"""Whether this distribution is installed with the ``.egg-info`` format.
This usually indicates the distribution was installed with setuptools
with an old pip version or with ``single-version-externally-managed``.
Note that this ensure the metadata store is a directory. distutils can
also installs an ``.egg-info``, but as a file, not a directory. This
property is *False* for that case. Also see ``installed_by_distutils``.
"""
info_location = self.info_location
if not info_location:
return False
if not info_location.endswith(".egg-info"):
return False
return pathlib.Path(info_location).is_dir()
@property
def installed_with_dist_info(self) -> bool:
"""Whether this distribution is installed with the "modern format".
This indicates a "modern" installation, e.g. storing metadata in the
``.dist-info`` directory. This applies to installations made by
setuptools (but through pip, not directly), or anything using the
standardized build backend interface (PEP 517).
"""
info_location = self.info_location
if not info_location:
return False
if not info_location.endswith(".dist-info"):
return False
return pathlib.Path(info_location).is_dir()
@property
def canonical_name(self) -> NormalizedName:
raise NotImplementedError()
@@ -240,14 +120,6 @@ class BaseDistribution(Protocol):
def version(self) -> DistributionVersion:
raise NotImplementedError()
@property
def setuptools_filename(self) -> str:
"""Convert a project name to its setuptools-compatible filename.
This is a copy of ``pkg_resources.to_filename()`` for compatibility.
"""
return self.raw_name.replace("-", "_")
@property
def direct_url(self) -> Optional[DirectUrl]:
"""Obtain a DirectUrl from this distribution.
@@ -276,15 +148,7 @@ class BaseDistribution(Protocol):
@property
def installer(self) -> str:
try:
installer_text = self.read_text("INSTALLER")
except (OSError, ValueError, NoneMetadataError):
return "" # Fail silently if the installer file cannot be read.
for line in installer_text.splitlines():
cleaned_line = line.strip()
if cleaned_line:
return cleaned_line
return ""
raise NotImplementedError()
@property
def editable(self) -> bool:
@@ -292,46 +156,21 @@ class BaseDistribution(Protocol):
@property
def local(self) -> bool:
"""If distribution is installed in the current virtual environment.
Always True if we're not in a virtualenv.
"""
if self.installed_location is None:
return False
return is_local(self.installed_location)
raise NotImplementedError()
@property
def in_usersite(self) -> bool:
if self.installed_location is None or user_site is None:
return False
return self.installed_location.startswith(normalize_path(user_site))
raise NotImplementedError()
@property
def in_site_packages(self) -> bool:
if self.installed_location is None or site_packages is None:
return False
return self.installed_location.startswith(normalize_path(site_packages))
def is_file(self, path: InfoPath) -> bool:
"""Check whether an entry in the info directory is a file."""
raise NotImplementedError()
def iterdir(self, path: InfoPath) -> Iterator[pathlib.PurePosixPath]:
"""Iterate through a directory in the info directory.
def read_text(self, name: str) -> str:
"""Read a file in the .dist-info (or .egg-info) directory.
Each item yielded would be a path relative to the info directory.
:raise FileNotFoundError: If ``name`` does not exist in the directory.
:raise NotADirectoryError: If ``name`` does not point to a directory.
"""
raise NotImplementedError()
def read_text(self, path: InfoPath) -> str:
"""Read a file in the info directory.
:raise FileNotFoundError: If ``name`` does not exist in the directory.
:raise NoneMetadataError: If ``name`` exists in the info directory, but
cannot be read.
Should raise ``FileNotFoundError`` if ``name`` does not exist in the
metadata directory.
"""
raise NotImplementedError()
@@ -340,13 +179,7 @@ class BaseDistribution(Protocol):
@property
def metadata(self) -> email.message.Message:
"""Metadata of distribution parsed from e.g. METADATA or PKG-INFO.
This should return an empty message if the metadata file is unavailable.
:raises NoneMetadataError: If the metadata file is available, but does
not contain valid metadata.
"""
"""Metadata of distribution parsed from e.g. METADATA or PKG-INFO."""
raise NotImplementedError()
@property
@@ -396,51 +229,6 @@ class BaseDistribution(Protocol):
"""
raise NotImplementedError()
def _iter_declared_entries_from_record(self) -> Optional[Iterator[str]]:
try:
text = self.read_text("RECORD")
except FileNotFoundError:
return None
# This extra Path-str cast normalizes entries.
return (str(pathlib.Path(row[0])) for row in csv.reader(text.splitlines()))
def _iter_declared_entries_from_legacy(self) -> Optional[Iterator[str]]:
try:
text = self.read_text("installed-files.txt")
except FileNotFoundError:
return None
paths = (p for p in text.splitlines(keepends=False) if p)
root = self.location
info = self.info_location
if root is None or info is None:
return paths
try:
info_rel = pathlib.Path(info).relative_to(root)
except ValueError: # info is not relative to root.
return paths
if not info_rel.parts: # info *is* root.
return paths
return (
_convert_installed_files_path(pathlib.Path(p).parts, info_rel.parts)
for p in paths
)
def iter_declared_entries(self) -> Optional[Iterator[str]]:
"""Iterate through file entires declared in this distribution.
For modern .dist-info distributions, this is the files listed in the
``RECORD`` metadata file. For legacy setuptools distributions, this
comes from ``installed-files.txt``, with entries normalized to be
compatible with the format used by ``RECORD``.
:return: An iterator for listed entries, or None if the distribution
contains neither ``RECORD`` nor ``installed-files.txt``.
"""
return (
self._iter_declared_entries_from_record()
or self._iter_declared_entries_from_legacy()
)
class BaseEnvironment:
"""An environment containing distributions to introspect."""
@@ -454,11 +242,7 @@ class BaseEnvironment:
raise NotImplementedError()
def get_distribution(self, name: str) -> Optional["BaseDistribution"]:
"""Given a requirement name, return the installed distributions.
The name may not be normalized. The implementation must canonicalize
it for lookup.
"""
"""Given a requirement name, return the installed distributions."""
raise NotImplementedError()
def _iter_distributions(self) -> Iterator["BaseDistribution"]:
@@ -1,26 +1,21 @@
import email.message
import email.parser
import logging
import os
import pathlib
import zipfile
from typing import Collection, Iterable, Iterator, List, Mapping, NamedTuple, Optional
from typing import Collection, Iterable, Iterator, List, NamedTuple, Optional
from pip._vendor import pkg_resources
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import parse as parse_version
from pip._internal.exceptions import InvalidWheel, NoneMetadataError, UnsupportedWheel
from pip._internal.utils.misc import display_path
from pip._internal.utils.wheel import parse_wheel, read_wheel_metadata_file
from pip._internal.utils import misc # TODO: Move definition here.
from pip._internal.utils.packaging import get_installer, get_metadata
from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
from .base import (
BaseDistribution,
BaseEntryPoint,
BaseEnvironment,
DistributionVersion,
InfoPath,
Wheel,
)
@@ -33,91 +28,14 @@ class EntryPoint(NamedTuple):
group: str
class WheelMetadata:
"""IMetadataProvider that reads metadata files from a dictionary.
This also maps metadata decoding exceptions to our internal exception type.
"""
def __init__(self, metadata: Mapping[str, bytes], wheel_name: str) -> None:
self._metadata = metadata
self._wheel_name = wheel_name
def has_metadata(self, name: str) -> bool:
return name in self._metadata
def get_metadata(self, name: str) -> str:
try:
return self._metadata[name].decode()
except UnicodeDecodeError as e:
# Augment the default error with the origin of the file.
raise UnsupportedWheel(
f"Error decoding metadata for {self._wheel_name}: {e} in {name} file"
)
def get_metadata_lines(self, name: str) -> Iterable[str]:
return pkg_resources.yield_lines(self.get_metadata(name))
def metadata_isdir(self, name: str) -> bool:
return False
def metadata_listdir(self, name: str) -> List[str]:
return []
def run_script(self, script_name: str, namespace: str) -> None:
pass
class Distribution(BaseDistribution):
def __init__(self, dist: pkg_resources.Distribution) -> None:
self._dist = dist
@classmethod
def from_directory(cls, directory: str) -> "Distribution":
dist_dir = directory.rstrip(os.sep)
# Build a PathMetadata object, from path to metadata. :wink:
base_dir, dist_dir_name = os.path.split(dist_dir)
metadata = pkg_resources.PathMetadata(base_dir, dist_dir)
# Determine the correct Distribution object type.
if dist_dir.endswith(".egg-info"):
dist_cls = pkg_resources.Distribution
dist_name = os.path.splitext(dist_dir_name)[0]
else:
assert dist_dir.endswith(".dist-info")
dist_cls = pkg_resources.DistInfoDistribution
dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
dist = dist_cls(base_dir, project_name=dist_name, metadata=metadata)
return cls(dist)
@classmethod
def from_wheel(cls, wheel: Wheel, name: str) -> "Distribution":
"""Load the distribution from a given wheel.
:raises InvalidWheel: Whenever loading of the wheel causes a
:py:exc:`zipfile.BadZipFile` exception to be thrown.
:raises UnsupportedWheel: If the wheel is a valid zip, but malformed
internally.
"""
try:
with wheel.as_zipfile() as zf:
info_dir, _ = parse_wheel(zf, name)
metadata_text = {
path.split("/", 1)[-1]: read_wheel_metadata_file(zf, path)
for path in zf.namelist()
if path.startswith(f"{info_dir}/")
}
except zipfile.BadZipFile as e:
raise InvalidWheel(wheel.location, name) from e
except UnsupportedWheel as e:
raise UnsupportedWheel(f"{name} has an invalid wheel, {e}")
dist = pkg_resources.DistInfoDistribution(
location=wheel.location,
metadata=WheelMetadata(metadata_text, wheel.location),
project_name=name,
)
with wheel.as_zipfile() as zf:
dist = pkg_resources_distribution_for_wheel(zf, name, wheel.location)
return cls(dist)
@property
@@ -125,19 +43,9 @@ class Distribution(BaseDistribution):
return self._dist.location
@property
def info_location(self) -> Optional[str]:
def info_directory(self) -> Optional[str]:
return self._dist.egg_info
@property
def installed_by_distutils(self) -> bool:
# A distutils-installed distribution is provided by FileMetadata. This
# provider has a "path" attribute not present anywhere else. Not the
# best introspection logic, but pip has been doing this for a long time.
try:
return bool(self._dist._provider.path)
except AttributeError:
return False
@property
def canonical_name(self) -> NormalizedName:
return canonicalize_name(self._dist.project_name)
@@ -146,26 +54,26 @@ class Distribution(BaseDistribution):
def version(self) -> DistributionVersion:
return parse_version(self._dist.version)
def is_file(self, path: InfoPath) -> bool:
return self._dist.has_metadata(str(path))
@property
def installer(self) -> str:
return get_installer(self._dist)
def iterdir(self, path: InfoPath) -> Iterator[pathlib.PurePosixPath]:
name = str(path)
@property
def local(self) -> bool:
return misc.dist_is_local(self._dist)
@property
def in_usersite(self) -> bool:
return misc.dist_in_usersite(self._dist)
@property
def in_site_packages(self) -> bool:
return misc.dist_in_site_packages(self._dist)
def read_text(self, name: str) -> str:
if not self._dist.has_metadata(name):
raise FileNotFoundError(name)
if not self._dist.isdir(name):
raise NotADirectoryError(name)
for child in self._dist.metadata_listdir(name):
yield pathlib.PurePosixPath(path, child)
def read_text(self, path: InfoPath) -> str:
name = str(path)
if not self._dist.has_metadata(name):
raise FileNotFoundError(name)
content = self._dist.get_metadata(name)
if content is None:
raise NoneMetadataError(self, name)
return content
return self._dist.get_metadata(name)
def iter_entry_points(self) -> Iterable[BaseEntryPoint]:
for group, entries in self._dist.get_entry_map().items():
@@ -175,26 +83,7 @@ class Distribution(BaseDistribution):
@property
def metadata(self) -> email.message.Message:
"""
:raises NoneMetadataError: if the distribution reports `has_metadata()`
True but `get_metadata()` returns None.
"""
if isinstance(self._dist, pkg_resources.DistInfoDistribution):
metadata_name = "METADATA"
else:
metadata_name = "PKG-INFO"
try:
metadata = self.read_text(metadata_name)
except FileNotFoundError:
if self.location:
displaying_path = display_path(self.location)
else:
displaying_path = repr(self.location)
logger.warning("No metadata found in %s", displaying_path)
metadata = ""
feed_parser = email.parser.FeedParser()
feed_parser.feed(metadata)
return feed_parser.close()
return get_metadata(self._dist)
def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
if extras: # pkg_resources raises on invalid extras, so we sanitize.
@@ -230,6 +119,7 @@ class Environment(BaseEnvironment):
return None
def get_distribution(self, name: str) -> Optional[BaseDistribution]:
# Search the distribution by looking through the working set.
dist = self._search_distribution(name)
if dist:
@@ -53,7 +53,7 @@ class SafeFileCache(BaseCache):
with open(path, "rb") as f:
return f.read()
def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None:
def set(self, key: str, value: bytes) -> None:
path = self._get_cache_path(key)
with suppressed_cache_errors():
ensure_dir(os.path.dirname(path))
@@ -8,7 +8,7 @@ from typing import Iterable, Optional, Tuple
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
from pip._internal.cli.progress_bars import get_download_progress_renderer
from pip._internal.cli.progress_bars import DownloadProgressProvider
from pip._internal.exceptions import NetworkConnectionError
from pip._internal.models.index import PyPI
from pip._internal.models.link import Link
@@ -65,8 +65,7 @@ def _prepare_download(
if not show_progress:
return chunks
renderer = get_download_progress_renderer(bar_type=progress_bar, size=total_length)
return renderer(chunks)
return DownloadProgressProvider(progress_bar, max=total_length)(chunks)
def sanitize_content_filename(filename: str) -> str:
@@ -6,17 +6,11 @@ import os
from pip._vendor.pep517.wrappers import Pep517HookCaller
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import (
InstallationSubprocessError,
MetadataGenerationFailed,
)
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory
def generate_metadata(
build_env: BuildEnvironment, backend: Pep517HookCaller, details: str
) -> str:
def generate_metadata(build_env: BuildEnvironment, backend: Pep517HookCaller) -> str:
"""Generate metadata using mechanisms described in PEP 517.
Returns the generated metadata directory.
@@ -31,9 +25,6 @@ def generate_metadata(
# consider the possibility that this hook doesn't exist.
runner = runner_with_spinner_message("Preparing metadata (pyproject.toml)")
with backend.subprocess_runner(runner):
try:
distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
except InstallationSubprocessError as error:
raise MetadataGenerationFailed(package_details=details) from error
distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
return os.path.join(metadata_dir, distinfo_dir)
@@ -6,16 +6,12 @@ import os
from pip._vendor.pep517.wrappers import Pep517HookCaller
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import (
InstallationSubprocessError,
MetadataGenerationFailed,
)
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory
def generate_editable_metadata(
build_env: BuildEnvironment, backend: Pep517HookCaller, details: str
build_env: BuildEnvironment, backend: Pep517HookCaller
) -> str:
"""Generate metadata using mechanisms described in PEP 660.
@@ -33,9 +29,6 @@ def generate_editable_metadata(
"Preparing editable metadata (pyproject.toml)"
)
with backend.subprocess_runner(runner):
try:
distinfo_dir = backend.prepare_metadata_for_build_editable(metadata_dir)
except InstallationSubprocessError as error:
raise MetadataGenerationFailed(package_details=details) from error
distinfo_dir = backend.prepare_metadata_for_build_editable(metadata_dir)
return os.path.join(metadata_dir, distinfo_dir)
@@ -6,11 +6,7 @@ import os
from pip._internal.build_env import BuildEnvironment
from pip._internal.cli.spinners import open_spinner
from pip._internal.exceptions import (
InstallationError,
InstallationSubprocessError,
MetadataGenerationFailed,
)
from pip._internal.exceptions import InstallationError
from pip._internal.utils.setuptools_build import make_setuptools_egg_info_args
from pip._internal.utils.subprocess import call_subprocess
from pip._internal.utils.temp_dir import TempDirectory
@@ -60,15 +56,12 @@ def generate_metadata(
with build_env:
with open_spinner("Preparing metadata (setup.py)") as spinner:
try:
call_subprocess(
args,
cwd=source_dir,
command_desc="python setup.py egg_info",
spinner=spinner,
)
except InstallationSubprocessError as error:
raise MetadataGenerationFailed(package_details=details) from error
call_subprocess(
args,
cwd=source_dir,
command_desc="python setup.py egg_info",
spinner=spinner,
)
# Return the .egg-info directory.
return _find_egg_info(egg_info_dir)
@@ -4,7 +4,11 @@ from typing import List, Optional
from pip._internal.cli.spinners import open_spinner
from pip._internal.utils.setuptools_build import make_setuptools_bdist_wheel_args
from pip._internal.utils.subprocess import call_subprocess, format_command_args
from pip._internal.utils.subprocess import (
LOG_DIVIDER,
call_subprocess,
format_command_args,
)
logger = logging.getLogger(__name__)
@@ -24,7 +28,7 @@ def format_command_result(
else:
if not command_output.endswith("\n"):
command_output += "\n"
text += f"Command output:\n{command_output}"
text += f"Command output:\n{command_output}{LOG_DIVIDER}"
return text
@@ -82,7 +86,6 @@ def build_wheel_legacy(
try:
output = call_subprocess(
wheel_args,
command_desc="python setup.py bdist_wheel",
cwd=source_dir,
spinner=spinner,
)
@@ -42,6 +42,5 @@ def install_editable(
with build_env:
call_subprocess(
args,
command_desc="python setup.py develop",
cwd=unpacked_source_directory,
)
@@ -7,8 +7,9 @@ from distutils.util import change_root
from typing import List, Optional, Sequence
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import InstallationError, LegacyInstallFailure
from pip._internal.exceptions import InstallationError
from pip._internal.models.scheme import Scheme
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import ensure_dir
from pip._internal.utils.setuptools_build import make_setuptools_install_args
from pip._internal.utils.subprocess import runner_with_spinner_message
@@ -17,6 +18,10 @@ from pip._internal.utils.temp_dir import TempDirectory
logger = logging.getLogger(__name__)
class LegacyInstallFailure(Exception):
pass
def write_installed_files_from_setuptools_record(
record_lines: List[str],
root: Optional[str],
@@ -93,7 +98,7 @@ def install(
runner = runner_with_spinner_message(
f"Running setup.py install for {req_name}"
)
with build_env:
with indent_log(), build_env:
runner(
cmd=install_args,
cwd=unpacked_source_directory,
@@ -106,7 +111,7 @@ def install(
except Exception as e:
# Signal to the caller that we didn't install the new package
raise LegacyInstallFailure(package_details=req_name) from e
raise LegacyInstallFailure from e
# At this point, we have successfully installed the requirement.
@@ -59,10 +59,10 @@ def _get_prepared_distribution(
return abstract_dist.get_metadata_distribution()
def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
def unpack_vcs_link(link: Link, location: str) -> None:
vcs_backend = vcs.get_backend_for_scheme(link.scheme)
assert vcs_backend is not None
vcs_backend.unpack(location, url=hide_url(link.url), verbosity=verbosity)
vcs_backend.unpack(location, url=hide_url(link.url))
class File:
@@ -175,7 +175,6 @@ def unpack_url(
link: Link,
location: str,
download: Downloader,
verbosity: int,
download_dir: Optional[str] = None,
hashes: Optional[Hashes] = None,
) -> Optional[File]:
@@ -188,7 +187,7 @@ def unpack_url(
"""
# non-editable vcs urls
if link.is_vcs:
unpack_vcs_link(link, location, verbosity=verbosity)
unpack_vcs_link(link, location)
return None
# Once out-of-tree-builds are no longer supported, could potentially
@@ -268,7 +267,6 @@ class RequirementPreparer:
require_hashes: bool,
use_user_site: bool,
lazy_wheel: bool,
verbosity: int,
in_tree_build: bool,
) -> None:
super().__init__()
@@ -297,9 +295,6 @@ class RequirementPreparer:
# Should wheels be downloaded lazily?
self.use_lazy_wheel = lazy_wheel
# How verbose should underlying tooling be?
self.verbosity = verbosity
# Should in-tree builds be used for local paths?
self.in_tree_build = in_tree_build
@@ -530,12 +525,7 @@ class RequirementPreparer:
elif link.url not in self._downloaded:
try:
local_file = unpack_url(
link,
req.source_dir,
self._download,
self.verbosity,
self.download_dir,
hashes,
link, req.source_dir, self._download, self.download_dir, hashes
)
except NetworkConnectionError as exc:
raise InstallationError(
@@ -5,11 +5,7 @@ from typing import Any, List, Optional
from pip._vendor import tomli
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._internal.exceptions import (
InstallationError,
InvalidPyProjectBuildRequires,
MissingPyProjectBuildRequires,
)
from pip._internal.exceptions import InstallationError
def _is_list_of_str(obj: Any) -> bool:
@@ -60,7 +56,7 @@ def load_pyproject_toml(
if has_pyproject:
with open(pyproject_toml, encoding="utf-8") as f:
pp_toml = tomli.loads(f.read())
pp_toml = tomli.load(f)
build_system = pp_toml.get("build-system")
else:
build_system = None
@@ -123,28 +119,47 @@ def load_pyproject_toml(
# Ensure that the build-system section in pyproject.toml conforms
# to PEP 518.
error_template = (
"{package} has a pyproject.toml file that does not comply "
"with PEP 518: {reason}"
)
# Specifying the build-system table but not the requires key is invalid
if "requires" not in build_system:
raise MissingPyProjectBuildRequires(package=req_name)
raise InstallationError(
error_template.format(
package=req_name,
reason=(
"it has a 'build-system' table but not "
"'build-system.requires' which is mandatory in the table"
),
)
)
# Error out if requires is not a list of strings
requires = build_system["requires"]
if not _is_list_of_str(requires):
raise InvalidPyProjectBuildRequires(
package=req_name,
reason="It is not a list of strings.",
raise InstallationError(
error_template.format(
package=req_name,
reason="'build-system.requires' is not a list of strings.",
)
)
# Each requirement must be valid as per PEP 508
for requirement in requires:
try:
Requirement(requirement)
except InvalidRequirement as error:
raise InvalidPyProjectBuildRequires(
package=req_name,
reason=f"It contains an invalid requirement: {requirement!r}",
) from error
except InvalidRequirement:
raise InstallationError(
error_template.format(
package=req_name,
reason=(
"'build-system.requires' contains an invalid "
"requirement: {!r}".format(requirement)
),
)
)
backend = build_system.get("build-backend")
backend_path = build_system.get("backend-path", [])
@@ -16,6 +16,7 @@ from typing import Any, Dict, Optional, Set, Tuple, Union
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.specifiers import Specifier
from pip._vendor.pkg_resources import RequirementParseError, parse_requirements
from pip._internal.exceptions import InstallationError
from pip._internal.models.index import PyPI, TestPyPI
@@ -112,56 +113,31 @@ def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
return package_name, url, set()
def check_first_requirement_in_file(filename: str) -> None:
"""Check if file is parsable as a requirements file.
This is heavily based on ``pkg_resources.parse_requirements``, but
simplified to just check the first meaningful line.
:raises InvalidRequirement: If the first meaningful line cannot be parsed
as an requirement.
"""
with open(filename, encoding="utf-8", errors="ignore") as f:
# Create a steppable iterator, so we can handle \-continuations.
lines = (
line
for line in (line.strip() for line in f)
if line and not line.startswith("#") # Skip blank lines/comments.
)
for line in lines:
# Drop comments -- a hash without a space may be in a URL.
if " #" in line:
line = line[: line.find(" #")]
# If there is a line continuation, drop it, and append the next line.
if line.endswith("\\"):
line = line[:-2].strip() + next(lines, "")
Requirement(line)
return
def deduce_helpful_msg(req: str) -> str:
"""Returns helpful msg in case requirements file does not exist,
or cannot be parsed.
:params req: Requirements file path
"""
if not os.path.exists(req):
return f" File '{req}' does not exist."
msg = " The path does exist. "
# Try to parse and check if it is a requirements file.
try:
check_first_requirement_in_file(req)
except InvalidRequirement:
logger.debug("Cannot parse '%s' as requirements file", req)
msg = ""
if os.path.exists(req):
msg = " The path does exist. "
# Try to parse and check if it is a requirements file.
try:
with open(req) as fp:
# parse first line only
next(parse_requirements(fp.read()))
msg += (
"The argument you provided "
"({}) appears to be a"
" requirements file. If that is the"
" case, use the '-r' flag to install"
" the packages specified within it."
).format(req)
except RequirementParseError:
logger.debug("Cannot parse '%s' as requirements file", req, exc_info=True)
else:
msg += (
f"The argument you provided "
f"({req}) appears to be a"
f" requirements file. If that is the"
f" case, use the '-r' flag to install"
f" the packages specified within it."
)
msg += f" File '{req}' does not exist."
return msg
@@ -10,6 +10,7 @@ import uuid
import zipfile
from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, Union
from pip._vendor import pkg_resources
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.specifiers import SpecifierSet
@@ -17,15 +18,11 @@ from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
from pip._vendor.pep517.wrappers import Pep517HookCaller
from pip._vendor.pkg_resources import Distribution
from pip._internal.build_env import BuildEnvironment, NoOpBuildEnvironment
from pip._internal.exceptions import InstallationError, LegacyInstallFailure
from pip._internal.exceptions import InstallationError
from pip._internal.locations import get_scheme
from pip._internal.metadata import (
BaseDistribution,
get_default_environment,
get_directory_distribution,
)
from pip._internal.models.link import Link
from pip._internal.operations.build.metadata import generate_metadata
from pip._internal.operations.build.metadata_editable import generate_editable_metadata
@@ -35,6 +32,7 @@ from pip._internal.operations.build.metadata_legacy import (
from pip._internal.operations.install.editable_legacy import (
install_editable as install_editable_legacy,
)
from pip._internal.operations.install.legacy import LegacyInstallFailure
from pip._internal.operations.install.legacy import install as install_legacy
from pip._internal.operations.install.wheel import install_wheel
from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path
@@ -49,10 +47,13 @@ from pip._internal.utils.misc import (
ask_path_exists,
backup_dir,
display_path,
dist_in_site_packages,
dist_in_usersite,
get_distribution,
hide_url,
redact_auth_from_url,
)
from pip._internal.utils.packaging import safe_extra
from pip._internal.utils.packaging import get_metadata
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
from pip._internal.utils.virtualenv import running_under_virtualenv
@@ -61,6 +62,32 @@ from pip._internal.vcs import vcs
logger = logging.getLogger(__name__)
def _get_dist(metadata_directory: str) -> Distribution:
"""Return a pkg_resources.Distribution for the provided
metadata directory.
"""
dist_dir = metadata_directory.rstrip(os.sep)
# Build a PathMetadata object, from path to metadata. :wink:
base_dir, dist_dir_name = os.path.split(dist_dir)
metadata = pkg_resources.PathMetadata(base_dir, dist_dir)
# Determine the correct Distribution object type.
if dist_dir.endswith(".egg-info"):
dist_cls = pkg_resources.Distribution
dist_name = os.path.splitext(dist_dir_name)[0]
else:
assert dist_dir.endswith(".dist-info")
dist_cls = pkg_resources.DistInfoDistribution
dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
return dist_cls(
base_dir,
project_name=dist_name,
metadata=metadata,
)
class InstallRequirement:
"""
Represents something that may be installed later on, may have information
@@ -118,15 +145,16 @@ class InstallRequirement:
if extras:
self.extras = extras
elif req:
self.extras = {safe_extra(extra) for extra in req.extras}
self.extras = {pkg_resources.safe_extra(extra) for extra in req.extras}
else:
self.extras = set()
if markers is None and req:
markers = req.marker
self.markers = markers
# This holds the Distribution object if this requirement is already installed.
self.satisfied_by: Optional[BaseDistribution] = None
# This holds the pkg_resources.Distribution object if this requirement
# is already available:
self.satisfied_by: Optional[Distribution] = None
# Whether the installation process should try to uninstall an existing
# distribution before installing this requirement.
self.should_reinstall = False
@@ -214,7 +242,7 @@ class InstallRequirement:
def name(self) -> Optional[str]:
if self.req is None:
return None
return self.req.name
return pkg_resources.safe_name(self.req.name)
@functools.lru_cache() # use cached_property in python 3.8+
def supports_pyproject_editable(self) -> bool:
@@ -387,24 +415,32 @@ class InstallRequirement:
"""
if self.req is None:
return
existing_dist = get_default_environment().get_distribution(self.req.name)
existing_dist = get_distribution(self.req.name)
if not existing_dist:
return
version_compatible = self.req.specifier.contains(
existing_dist.version,
prereleases=True,
# pkg_resouces may contain a different copy of packaging.version from
# pip in if the downstream distributor does a poor job debundling pip.
# We avoid existing_dist.parsed_version and let SpecifierSet.contains
# parses the version instead.
existing_version = existing_dist.version
version_compatible = (
existing_version is not None
and self.req.specifier.contains(existing_version, prereleases=True)
)
if not version_compatible:
self.satisfied_by = None
if use_user_site:
if existing_dist.in_usersite:
if dist_in_usersite(existing_dist):
self.should_reinstall = True
elif running_under_virtualenv() and existing_dist.in_site_packages:
elif running_under_virtualenv() and dist_in_site_packages(
existing_dist
):
raise InstallationError(
f"Will not install to the user site because it will "
f"lack sys.path precedence to {existing_dist.raw_name} "
f"in {existing_dist.location}"
"Will not install to the user site because it will "
"lack sys.path precedence to {} in {}".format(
existing_dist.project_name, existing_dist.location
)
)
else:
self.should_reinstall = True
@@ -504,7 +540,6 @@ class InstallRequirement:
Under legacy processing, call setup.py egg-info.
"""
assert self.source_dir
details = self.name or f"from {self.link}"
if self.use_pep517:
assert self.pep517_backend is not None
@@ -516,13 +551,11 @@ class InstallRequirement:
self.metadata_directory = generate_editable_metadata(
build_env=self.build_env,
backend=self.pep517_backend,
details=details,
)
else:
self.metadata_directory = generate_metadata(
build_env=self.build_env,
backend=self.pep517_backend,
details=details,
)
else:
self.metadata_directory = generate_metadata_legacy(
@@ -530,7 +563,7 @@ class InstallRequirement:
setup_py_path=self.setup_py_path,
source_dir=self.unpacked_source_directory,
isolated=self.isolated,
details=details,
details=self.name or f"from {self.link}",
)
# Act on the newly generated metadata, based on the name and version.
@@ -544,12 +577,12 @@ class InstallRequirement:
@property
def metadata(self) -> Any:
if not hasattr(self, "_metadata"):
self._metadata = self.get_dist().metadata
self._metadata = get_metadata(self.get_dist())
return self._metadata
def get_dist(self) -> BaseDistribution:
return get_directory_distribution(self.metadata_directory)
def get_dist(self) -> Distribution:
return _get_dist(self.metadata_directory)
def assert_source_matches_version(self) -> None:
assert self.source_dir
@@ -609,7 +642,7 @@ class InstallRequirement:
# So here, if it's neither a path nor a valid VCS URL, it's a bug.
assert vcs_backend, f"Unsupported VCS URL {self.link.url}"
hidden_url = hide_url(self.link.url)
vcs_backend.obtain(self.source_dir, url=hidden_url, verbosity=0)
vcs_backend.obtain(self.source_dir, url=hidden_url)
# Top-level Actions
def uninstall(
@@ -628,7 +661,7 @@ class InstallRequirement:
"""
assert self.req
dist = get_default_environment().get_distribution(self.req.name)
dist = get_distribution(self.req.name)
if not dist:
logger.warning("Skipping %s as it is not installed.", self.name)
return None
@@ -808,7 +841,7 @@ class InstallRequirement:
)
except LegacyInstallFailure as exc:
self.install_succeeded = False
raise exc
raise exc.__cause__
except Exception:
self.install_succeeded = True
raise
@@ -1,3 +1,4 @@
import csv
import functools
import os
import sys
@@ -5,33 +6,47 @@ import sysconfig
from importlib.util import cache_from_source
from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple
from pip._vendor import pkg_resources
from pip._vendor.pkg_resources import Distribution
from pip._internal.exceptions import UninstallationError
from pip._internal.locations import get_bin_prefix, get_bin_user
from pip._internal.metadata import BaseDistribution
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.egg_link import egg_link_path_from_location
from pip._internal.utils.logging import getLogger, indent_log
from pip._internal.utils.misc import ask, is_local, normalize_path, renames, rmtree
from pip._internal.utils.misc import (
ask,
dist_in_usersite,
dist_is_local,
is_local,
normalize_path,
renames,
rmtree,
)
from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory
logger = getLogger(__name__)
def _script_names(bin_dir: str, script_name: str, is_gui: bool) -> Iterator[str]:
def _script_names(dist: Distribution, script_name: str, is_gui: bool) -> List[str]:
"""Create the fully qualified name of the files created by
{console,gui}_scripts for the given ``dist``.
Returns the list of file names
"""
exe_name = os.path.join(bin_dir, script_name)
yield exe_name
if not WINDOWS:
return
yield f"{exe_name}.exe"
yield f"{exe_name}.exe.manifest"
if is_gui:
yield f"{exe_name}-script.pyw"
if dist_in_usersite(dist):
bin_dir = get_bin_user()
else:
yield f"{exe_name}-script.py"
bin_dir = get_bin_prefix()
exe_name = os.path.join(bin_dir, script_name)
paths_to_remove = [exe_name]
if WINDOWS:
paths_to_remove.append(exe_name + ".exe")
paths_to_remove.append(exe_name + ".exe.manifest")
if is_gui:
paths_to_remove.append(exe_name + "-script.pyw")
else:
paths_to_remove.append(exe_name + "-script.py")
return paths_to_remove
def _unique(fn: Callable[..., Iterator[Any]]) -> Callable[..., Iterator[Any]]:
@@ -47,7 +62,7 @@ def _unique(fn: Callable[..., Iterator[Any]]) -> Callable[..., Iterator[Any]]:
@_unique
def uninstallation_paths(dist: BaseDistribution) -> Iterator[str]:
def uninstallation_paths(dist: Distribution) -> Iterator[str]:
"""
Yield all the uninstallation paths for dist based on RECORD-without-.py[co]
@@ -61,25 +76,25 @@ def uninstallation_paths(dist: BaseDistribution) -> Iterator[str]:
https://packaging.python.org/specifications/recording-installed-packages/
"""
location = dist.location
assert location is not None, "not installed"
entries = dist.iter_declared_entries()
if entries is None:
try:
r = csv.reader(dist.get_metadata_lines("RECORD"))
except FileNotFoundError as missing_record_exception:
msg = "Cannot uninstall {dist}, RECORD file not found.".format(dist=dist)
installer = dist.installer
if not installer or installer == "pip":
dep = "{}=={}".format(dist.raw_name, dist.version)
try:
installer = next(dist.get_metadata_lines("INSTALLER"))
if not installer or installer == "pip":
raise ValueError()
except (OSError, StopIteration, ValueError):
dep = "{}=={}".format(dist.project_name, dist.version)
msg += (
" You might be able to recover from this via: "
"'pip install --force-reinstall --no-deps {}'.".format(dep)
)
else:
msg += " Hint: The package was installed by {}.".format(installer)
raise UninstallationError(msg)
for entry in entries:
path = os.path.join(location, entry)
raise UninstallationError(msg) from missing_record_exception
for row in r:
path = os.path.join(dist.location, row[0])
yield path
if path.endswith(".py"):
dn, fn = os.path.split(path)
@@ -302,11 +317,11 @@ class UninstallPathSet:
"""A set of file paths to be removed in the uninstallation of a
requirement."""
def __init__(self, dist: BaseDistribution) -> None:
self._paths: Set[str] = set()
def __init__(self, dist: Distribution) -> None:
self.paths: Set[str] = set()
self._refuse: Set[str] = set()
self._pth: Dict[str, UninstallPthEntries] = {}
self._dist = dist
self.pth: Dict[str, UninstallPthEntries] = {}
self.dist = dist
self._moved_paths = StashedUninstallPathSet()
def _permitted(self, path: str) -> bool:
@@ -327,7 +342,7 @@ class UninstallPathSet:
if not os.path.exists(path):
return
if self._permitted(path):
self._paths.add(path)
self.paths.add(path)
else:
self._refuse.add(path)
@@ -339,37 +354,37 @@ class UninstallPathSet:
def add_pth(self, pth_file: str, entry: str) -> None:
pth_file = normalize_path(pth_file)
if self._permitted(pth_file):
if pth_file not in self._pth:
self._pth[pth_file] = UninstallPthEntries(pth_file)
self._pth[pth_file].add(entry)
if pth_file not in self.pth:
self.pth[pth_file] = UninstallPthEntries(pth_file)
self.pth[pth_file].add(entry)
else:
self._refuse.add(pth_file)
def remove(self, auto_confirm: bool = False, verbose: bool = False) -> None:
"""Remove paths in ``self._paths`` with confirmation (unless
"""Remove paths in ``self.paths`` with confirmation (unless
``auto_confirm`` is True)."""
if not self._paths:
if not self.paths:
logger.info(
"Can't uninstall '%s'. No files were found to uninstall.",
self._dist.raw_name,
self.dist.project_name,
)
return
dist_name_version = f"{self._dist.raw_name}-{self._dist.version}"
dist_name_version = self.dist.project_name + "-" + self.dist.version
logger.info("Uninstalling %s:", dist_name_version)
with indent_log():
if auto_confirm or self._allowed_to_proceed(verbose):
moved = self._moved_paths
for_rename = compress_for_rename(self._paths)
for_rename = compress_for_rename(self.paths)
for path in sorted(compact(for_rename)):
moved.stash(path)
logger.verbose("Removing file or directory %s", path)
for pth in self._pth.values():
for pth in self.pth.values():
pth.remove()
logger.info("Successfully uninstalled %s", dist_name_version)
@@ -387,18 +402,18 @@ class UninstallPathSet:
logger.info(path)
if not verbose:
will_remove, will_skip = compress_for_output_listing(self._paths)
will_remove, will_skip = compress_for_output_listing(self.paths)
else:
# In verbose mode, display all the files that are going to be
# deleted.
will_remove = set(self._paths)
will_remove = set(self.paths)
will_skip = set()
_display("Would remove:", will_remove)
_display("Would not remove (might be manually added):", will_skip)
_display("Would not remove (outside of prefix):", self._refuse)
if verbose:
_display("Will actually move:", compress_for_rename(self._paths))
_display("Will actually move:", compress_for_rename(self.paths))
return ask("Proceed (Y/n)? ", ("y", "n", "")) != "n"
@@ -407,12 +422,12 @@ class UninstallPathSet:
if not self._moved_paths.can_rollback:
logger.error(
"Can't roll back %s; was not uninstalled",
self._dist.raw_name,
self.dist.project_name,
)
return
logger.info("Rolling back uninstall of %s", self._dist.raw_name)
logger.info("Rolling back uninstall of %s", self.dist.project_name)
self._moved_paths.rollback()
for pth in self._pth.values():
for pth in self.pth.values():
pth.rollback()
def commit(self) -> None:
@@ -420,156 +435,141 @@ class UninstallPathSet:
self._moved_paths.commit()
@classmethod
def from_dist(cls, dist: BaseDistribution) -> "UninstallPathSet":
dist_location = dist.location
info_location = dist.info_location
if dist_location is None:
logger.info(
"Not uninstalling %s since it is not installed",
dist.canonical_name,
)
return cls(dist)
normalized_dist_location = normalize_path(dist_location)
if not dist.local:
def from_dist(cls, dist: Distribution) -> "UninstallPathSet":
dist_path = normalize_path(dist.location)
if not dist_is_local(dist):
logger.info(
"Not uninstalling %s at %s, outside environment %s",
dist.canonical_name,
normalized_dist_location,
dist.key,
dist_path,
sys.prefix,
)
return cls(dist)
if normalized_dist_location in {
if dist_path in {
p
for p in {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")}
if p
}:
logger.info(
"Not uninstalling %s at %s, as it is in the standard library.",
dist.canonical_name,
normalized_dist_location,
dist.key,
dist_path,
)
return cls(dist)
paths_to_remove = cls(dist)
develop_egg_link = egg_link_path_from_location(dist.raw_name)
# Distribution is installed with metadata in a "flat" .egg-info
# directory. This means it is not a modern .dist-info installation, an
# egg, or legacy editable.
setuptools_flat_installation = (
dist.installed_with_setuptools_egg_info
and info_location is not None
and os.path.exists(info_location)
# If dist is editable and the location points to a ``.egg-info``,
# we are in fact in the legacy editable case.
and not info_location.endswith(f"{dist.setuptools_filename}.egg-info")
develop_egg_link = egg_link_path_from_location(dist.project_name)
develop_egg_link_egg_info = "{}.egg-info".format(
pkg_resources.to_filename(dist.project_name)
)
egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info)
# Special case for distutils installed package
distutils_egg_info = getattr(dist._provider, "path", None)
# Uninstall cases order do matter as in the case of 2 installs of the
# same package, pip needs to uninstall the currently detected version
if setuptools_flat_installation:
if info_location is not None:
paths_to_remove.add(info_location)
installed_files = dist.iter_declared_entries()
if installed_files is not None:
for installed_file in installed_files:
paths_to_remove.add(os.path.join(dist_location, installed_file))
if (
egg_info_exists
and dist.egg_info.endswith(".egg-info")
and not dist.egg_info.endswith(develop_egg_link_egg_info)
):
# if dist.egg_info.endswith(develop_egg_link_egg_info), we
# are in fact in the develop_egg_link case
paths_to_remove.add(dist.egg_info)
if dist.has_metadata("installed-files.txt"):
for installed_file in dist.get_metadata(
"installed-files.txt"
).splitlines():
path = os.path.normpath(os.path.join(dist.egg_info, installed_file))
paths_to_remove.add(path)
# FIXME: need a test for this elif block
# occurs with --single-version-externally-managed/--record outside
# of pip
elif dist.is_file("top_level.txt"):
try:
namespace_packages = dist.read_text("namespace_packages.txt")
except FileNotFoundError:
namespaces = []
elif dist.has_metadata("top_level.txt"):
if dist.has_metadata("namespace_packages.txt"):
namespaces = dist.get_metadata("namespace_packages.txt")
else:
namespaces = namespace_packages.splitlines(keepends=False)
namespaces = []
for top_level_pkg in [
p
for p in dist.read_text("top_level.txt").splitlines()
for p in dist.get_metadata("top_level.txt").splitlines()
if p and p not in namespaces
]:
path = os.path.join(dist_location, top_level_pkg)
path = os.path.join(dist.location, top_level_pkg)
paths_to_remove.add(path)
paths_to_remove.add(f"{path}.py")
paths_to_remove.add(f"{path}.pyc")
paths_to_remove.add(f"{path}.pyo")
paths_to_remove.add(path + ".py")
paths_to_remove.add(path + ".pyc")
paths_to_remove.add(path + ".pyo")
elif dist.installed_by_distutils:
elif distutils_egg_info:
raise UninstallationError(
"Cannot uninstall {!r}. It is a distutils installed project "
"and thus we cannot accurately determine which files belong "
"to it which would lead to only a partial uninstall.".format(
dist.raw_name,
dist.project_name,
)
)
elif dist.installed_as_egg:
elif dist.location.endswith(".egg"):
# package installed by easy_install
# We cannot match on dist.egg_name because it can slightly vary
# i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg
paths_to_remove.add(dist_location)
easy_install_egg = os.path.split(dist_location)[1]
paths_to_remove.add(dist.location)
easy_install_egg = os.path.split(dist.location)[1]
easy_install_pth = os.path.join(
os.path.dirname(dist_location),
"easy-install.pth",
os.path.dirname(dist.location), "easy-install.pth"
)
paths_to_remove.add_pth(easy_install_pth, "./" + easy_install_egg)
elif dist.installed_with_dist_info:
elif egg_info_exists and dist.egg_info.endswith(".dist-info"):
for path in uninstallation_paths(dist):
paths_to_remove.add(path)
elif develop_egg_link:
# PEP 660 modern editable is handled in the ``.dist-info`` case
# above, so this only covers the setuptools-style editable.
# develop egg
with open(develop_egg_link) as fh:
link_pointer = os.path.normcase(fh.readline().strip())
assert link_pointer == dist_location, (
f"Egg-link {link_pointer} does not match installed location of "
f"{dist.raw_name} (at {dist_location})"
assert (
link_pointer == dist.location
), "Egg-link {} does not match installed location of {} (at {})".format(
link_pointer, dist.project_name, dist.location
)
paths_to_remove.add(develop_egg_link)
easy_install_pth = os.path.join(
os.path.dirname(develop_egg_link), "easy-install.pth"
)
paths_to_remove.add_pth(easy_install_pth, dist_location)
paths_to_remove.add_pth(easy_install_pth, dist.location)
else:
logger.debug(
"Not sure how to uninstall: %s - Check: %s",
dist,
dist_location,
dist.location,
)
if dist.in_usersite:
bin_dir = get_bin_user()
else:
bin_dir = get_bin_prefix()
# find distutils scripts= scripts
try:
for script in dist.iterdir("scripts"):
paths_to_remove.add(os.path.join(bin_dir, script.name))
if dist.has_metadata("scripts") and dist.metadata_isdir("scripts"):
for script in dist.metadata_listdir("scripts"):
if dist_in_usersite(dist):
bin_dir = get_bin_user()
else:
bin_dir = get_bin_prefix()
paths_to_remove.add(os.path.join(bin_dir, script))
if WINDOWS:
paths_to_remove.add(os.path.join(bin_dir, f"{script.name}.bat"))
except (FileNotFoundError, NotADirectoryError):
pass
paths_to_remove.add(os.path.join(bin_dir, script) + ".bat")
# find console_scripts and gui_scripts
def iter_scripts_to_remove(
dist: BaseDistribution,
bin_dir: str,
) -> Iterator[str]:
for entry_point in dist.iter_entry_points():
if entry_point.group == "console_scripts":
yield from _script_names(bin_dir, entry_point.name, False)
elif entry_point.group == "gui_scripts":
yield from _script_names(bin_dir, entry_point.name, True)
# find console_scripts
_scripts_to_remove = []
console_scripts = dist.get_entry_map(group="console_scripts")
for name in console_scripts.keys():
_scripts_to_remove.extend(_script_names(dist, name, False))
# find gui_scripts
gui_scripts = dist.get_entry_map(group="gui_scripts")
for name in gui_scripts.keys():
_scripts_to_remove.extend(_script_names(dist, name, True))
for s in iter_scripts_to_remove(dist, bin_dir):
for s in _scripts_to_remove:
paths_to_remove.add(s)
return paths_to_remove
@@ -43,7 +43,7 @@ from pip._internal.req.req_set import RequirementSet
from pip._internal.resolution.base import BaseResolver, InstallRequirementProvider
from pip._internal.utils.compatibility_tags import get_supported
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import normalize_version_info
from pip._internal.utils.misc import dist_in_usersite, normalize_version_info
from pip._internal.utils.packaging import check_requires_python
logger = logging.getLogger(__name__)
@@ -203,7 +203,7 @@ class Resolver(BaseResolver):
"""
# Don't uninstall the conflict if doing a user install and the
# conflict is not a user install.
if not self.use_user_site or req.satisfied_by.in_usersite:
if not self.use_user_site or dist_in_usersite(req.satisfied_by):
req.should_reinstall = True
req.satisfied_by = None
@@ -5,11 +5,7 @@ from typing import TYPE_CHECKING, Any, FrozenSet, Iterable, Optional, Tuple, Uni
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._internal.exceptions import (
HashError,
InstallationSubprocessError,
MetadataInconsistent,
)
from pip._internal.exceptions import HashError, MetadataInconsistent
from pip._internal.metadata import BaseDistribution
from pip._internal.models.link import Link, links_equivalent
from pip._internal.models.wheel import Wheel
@@ -98,6 +94,8 @@ def make_install_req_from_editable(
def _make_install_req_from_dist(
dist: BaseDistribution, template: InstallRequirement
) -> InstallRequirement:
from pip._internal.metadata.pkg_resources import Distribution as _Dist
if template.req:
line = str(template.req)
elif template.link:
@@ -117,7 +115,7 @@ def _make_install_req_from_dist(
hashes=template.hash_options,
),
)
ireq.satisfied_by = dist
ireq.satisfied_by = cast(_Dist, dist)._dist
return ireq
@@ -231,11 +229,6 @@ class _InstallRequirementBackedCandidate(Candidate):
# offending line to the user.
e.req = self._ireq
raise
except InstallationSubprocessError as exc:
# The output has been presented already, so don't duplicate it.
exc.context = "See above for output."
raise
self._check_metadata_consistency(dist)
return dist
@@ -97,7 +97,6 @@ class Factory:
force_reinstall: bool,
ignore_installed: bool,
ignore_requires_python: bool,
suppress_build_failures: bool,
py_version_info: Optional[Tuple[int, ...]] = None,
) -> None:
self._finder = finder
@@ -108,7 +107,6 @@ class Factory:
self._use_user_site = use_user_site
self._force_reinstall = force_reinstall
self._ignore_requires_python = ignore_requires_python
self._suppress_build_failures = suppress_build_failures
self._build_failures: Cache[InstallationError] = {}
self._link_candidate_cache: Cache[LinkCandidate] = {}
@@ -192,22 +190,10 @@ class Factory:
name=name,
version=version,
)
except MetadataInconsistent as e:
logger.info(
"Discarding [blue underline]%s[/]: [yellow]%s[reset]",
link,
e,
extra={"markup": True},
)
except (InstallationSubprocessError, MetadataInconsistent) as e:
logger.warning("Discarding %s. %s", link, e)
self._build_failures[link] = e
return None
except InstallationSubprocessError as e:
if not self._suppress_build_failures:
raise
logger.warning("Discarding %s due to build failure: %s", link, e)
self._build_failures[link] = e
return None
base: BaseCandidate = self._editable_candidate_cache[link]
else:
if link not in self._link_candidate_cache:
@@ -219,19 +205,8 @@ class Factory:
name=name,
version=version,
)
except MetadataInconsistent as e:
logger.info(
"Discarding [blue underline]%s[/]: [yellow]%s[reset]",
link,
e,
extra={"markup": True},
)
self._build_failures[link] = e
return None
except InstallationSubprocessError as e:
if not self._suppress_build_failures:
raise
logger.warning("Discarding %s due to build failure: %s", link, e)
except (InstallationSubprocessError, MetadataInconsistent) as e:
logger.warning("Discarding %s. %s", link, e)
self._build_failures[link] = e
return None
base = self._link_candidate_cache[link]
@@ -285,7 +260,7 @@ class Factory:
extras=extras,
template=template,
)
# The candidate is a known incompatibility. Don't use it.
# The candidate is a known incompatiblity. Don't use it.
if id(candidate) in incompatible_ids:
return None
return candidate
@@ -298,27 +273,14 @@ class Factory:
)
icans = list(result.iter_applicable())
# PEP 592: Yanked releases are ignored unless the specifier
# explicitly pins a version (via '==' or '===') that can be
# solely satisfied by a yanked release.
# PEP 592: Yanked releases must be ignored unless only yanked
# releases can satisfy the version range. So if this is false,
# all yanked icans need to be skipped.
all_yanked = all(ican.link.is_yanked for ican in icans)
def is_pinned(specifier: SpecifierSet) -> bool:
for sp in specifier:
if sp.operator == "===":
return True
if sp.operator != "==":
continue
if sp.version.endswith(".*"):
continue
return True
return False
pinned = is_pinned(specifier)
# PackageFinder returns earlier versions first, so we reverse.
for ican in reversed(icans):
if not (all_yanked and pinned) and ican.link.is_yanked:
if not all_yanked and ican.link.is_yanked:
continue
func = functools.partial(
self._make_candidate_from_link,
@@ -412,7 +374,7 @@ class Factory:
)
# Add explicit candidates from constraints. We only do this if there are
# known ireqs, which represent requirements not already explicit. If
# kown ireqs, which represent requirements not already explicit. If
# there are no ireqs, we're constraining already-explicit requirements,
# which is handled later when we return the explicit candidates.
if ireqs:
@@ -653,7 +615,7 @@ class Factory:
]
if requires_python_causes:
# The comprehension above makes sure all Requirement instances are
# RequiresPythonRequirement, so let's cast for convenience.
# RequiresPythonRequirement, so let's cast for convinience.
return self._report_requires_python_error(
cast("Sequence[ConflictCause]", requires_python_causes),
)
@@ -734,6 +696,6 @@ class Factory:
return DistributionNotFound(
"ResolutionImpossible: for help visit "
"https://pip.pypa.io/en/latest/topics/dependency-resolution/"
"#dealing-with-dependency-conflicts"
"https://pip.pypa.io/en/latest/user_guide/"
"#fixing-conflicting-dependencies"
)
@@ -1,15 +1,6 @@
import collections
import math
from typing import (
TYPE_CHECKING,
Dict,
Iterable,
Iterator,
Mapping,
Sequence,
TypeVar,
Union,
)
from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Mapping, Sequence, Union
from pip._vendor.resolvelib.providers import AbstractProvider
@@ -46,35 +37,6 @@ else:
# services to those objects (access to pip's finder and preparer).
D = TypeVar("D")
V = TypeVar("V")
def _get_with_identifier(
mapping: Mapping[str, V],
identifier: str,
default: D,
) -> Union[D, V]:
"""Get item from a package name lookup mapping with a resolver identifier.
This extra logic is needed when the target mapping is keyed by package
name, which cannot be directly looked up with an identifier (which may
contain requested extras). Additional logic is added to also look up a value
by "cleaning up" the extras from the identifier.
"""
if identifier in mapping:
return mapping[identifier]
# HACK: Theoretically we should check whether this identifier is a valid
# "NAME[EXTRAS]" format, and parse out the name part with packaging or
# some regular expression. But since pip's resolver only spits out three
# kinds of identifiers: normalized PEP 503 names, normalized names plus
# extras, and Requires-Python, we can cheat a bit here.
name, open_bracket, _ = identifier.partition("[")
if open_bracket and name in mapping:
return mapping[name]
return default
class PipProvider(_ProviderBase):
"""Pip's provider implementation for resolvelib.
@@ -167,7 +129,7 @@ class PipProvider(_ProviderBase):
# (Most projects specify it only to request for an installer feature,
# which does not work, but that's another topic.) Intentionally
# delaying Setuptools helps reduce branches the resolver has to check.
# This serves as a temporary fix for issues like "apache-airflow[all]"
# This serves as a temporary fix for issues like "apache-airlfow[all]"
# while we work on "proper" branch pruning techniques.
delay_this = identifier == "setuptools"
@@ -188,13 +150,28 @@ class PipProvider(_ProviderBase):
identifier,
)
def _get_constraint(self, identifier: str) -> Constraint:
if identifier in self._constraints:
return self._constraints[identifier]
# HACK: Theoratically we should check whether this identifier is a valid
# "NAME[EXTRAS]" format, and parse out the name part with packaging or
# some regular expression. But since pip's resolver only spits out
# three kinds of identifiers: normalized PEP 503 names, normalized names
# plus extras, and Requires-Python, we can cheat a bit here.
name, open_bracket, _ = identifier.partition("[")
if open_bracket and name in self._constraints:
return self._constraints[name]
return Constraint.empty()
def find_matches(
self,
identifier: str,
requirements: Mapping[str, Iterator[Requirement]],
incompatibilities: Mapping[str, Iterator[Candidate]],
) -> Iterable[Candidate]:
def _eligible_for_upgrade(identifier: str) -> bool:
def _eligible_for_upgrade(name: str) -> bool:
"""Are upgrades allowed for this project?
This checks the upgrade strategy, and whether the project was one
@@ -208,23 +185,13 @@ class PipProvider(_ProviderBase):
if self._upgrade_strategy == "eager":
return True
elif self._upgrade_strategy == "only-if-needed":
user_order = _get_with_identifier(
self._user_requested,
identifier,
default=None,
)
return user_order is not None
return name in self._user_requested
return False
constraint = _get_with_identifier(
self._constraints,
identifier,
default=Constraint.empty(),
)
return self._factory.find_candidates(
identifier=identifier,
requirements=requirements,
constraint=constraint,
constraint=self._get_constraint(identifier),
prefers_installed=(not _eligible_for_upgrade(identifier)),
incompatibilities=incompatibilities,
)
@@ -21,12 +21,12 @@ class ExplicitRequirement(Requirement):
@property
def project_name(self) -> NormalizedName:
# No need to canonicalize - the candidate did this
# No need to canonicalise - the candidate did this
return self.candidate.project_name
@property
def name(self) -> str:
# No need to canonicalize - the candidate did this
# No need to canonicalise - the candidate did this
return self.candidate.name
def format_for_error(self) -> str:
@@ -47,7 +47,6 @@ class Resolver(BaseResolver):
ignore_requires_python: bool,
force_reinstall: bool,
upgrade_strategy: str,
suppress_build_failures: bool,
py_version_info: Optional[Tuple[int, ...]] = None,
):
super().__init__()
@@ -62,7 +61,6 @@ class Resolver(BaseResolver):
force_reinstall=force_reinstall,
ignore_installed=ignore_installed,
ignore_requires_python=ignore_requires_python,
suppress_build_failures=suppress_build_failures,
py_version_info=py_version_info,
)
self.ignore_dependencies = ignore_dependencies
@@ -173,19 +171,17 @@ class Resolver(BaseResolver):
get installed one-by-one.
The current implementation creates a topological ordering of the
dependency graph, giving more weight to packages with less
or no dependencies, while breaking any cycles in the graph at
arbitrary points. We make no guarantees about where the cycle
would be broken, other than it *would* be broken.
dependency graph, while breaking any cycles in the graph at arbitrary
points. We make no guarantees about where the cycle would be broken,
other than they would be broken.
"""
assert self._result is not None, "must call resolve() first"
if not req_set.requirements:
# Nothing is left to install, so we do not need an order.
return []
graph = self._result.graph
weights = get_topological_weights(graph, set(req_set.requirements.keys()))
weights = get_topological_weights(
graph,
expected_node_count=len(self._result.mapping) + 1,
)
sorted_items = sorted(
req_set.requirements.items(),
@@ -196,32 +192,23 @@ class Resolver(BaseResolver):
def get_topological_weights(
graph: "DirectedGraph[Optional[str]]", requirement_keys: Set[str]
graph: "DirectedGraph[Optional[str]]", expected_node_count: int
) -> Dict[Optional[str], int]:
"""Assign weights to each node based on how "deep" they are.
This implementation may change at any point in the future without prior
notice.
We first simplify the dependency graph by pruning any leaves and giving them
the highest weight: a package without any dependencies should be installed
first. This is done again and again in the same way, giving ever less weight
to the newly found leaves. The loop stops when no leaves are left: all
remaining packages have at least one dependency left in the graph.
Then we continue with the remaining graph, by taking the length for the
longest path to any node from root, ignoring any paths that contain a single
node twice (i.e. cycles). This is done through a depth-first search through
the graph, while keeping track of the path to the node.
We take the length for the longest path to any node from root, ignoring any
paths that contain a single node twice (i.e. cycles). This is done through
a depth-first search through the graph, while keeping track of the path to
the node.
Cycles in the graph result would result in node being revisited while also
being on its own path. In this case, take no action. This helps ensure we
being it's own path. In this case, take no action. This helps ensure we
don't get stuck in a cycle.
When assigning weight, the longer path (i.e. larger length) is preferred.
We are only interested in the weights of packages that are in the
requirement_keys.
"""
path: Set[Optional[str]] = set()
weights: Dict[Optional[str], int] = {}
@@ -237,49 +224,15 @@ def get_topological_weights(
visit(child)
path.remove(node)
if node not in requirement_keys:
return
last_known_parent_count = weights.get(node, 0)
weights[node] = max(last_known_parent_count, len(path))
# Simplify the graph, pruning leaves that have no dependencies.
# This is needed for large graphs (say over 200 packages) because the
# `visit` function is exponentially slower then, taking minutes.
# See https://github.com/pypa/pip/issues/10557
# We will loop until we explicitly break the loop.
while True:
leaves = set()
for key in graph:
if key is None:
continue
for _child in graph.iter_children(key):
# This means we have at least one child
break
else:
# No child.
leaves.add(key)
if not leaves:
# We are done simplifying.
break
# Calculate the weight for the leaves.
weight = len(graph) - 1
for leaf in leaves:
if leaf not in requirement_keys:
continue
weights[leaf] = weight
# Remove the leaves from the graph, making it simpler.
for leaf in leaves:
graph.remove(leaf)
# Visit the remaining graph.
# `None` is guaranteed to be the root node by resolvelib.
visit(None)
# Sanity check: all requirement keys should be in the weights,
# and no other keys should be in the weights.
difference = set(weights.keys()).difference(requirement_keys)
assert not difference, difference
# Sanity checks
assert weights[None] == 0
assert len(weights) == expected_node_count
return weights
@@ -141,9 +141,6 @@ def pip_self_version_check(session: PipSession, options: optparse.Values) -> Non
finder = PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
use_deprecated_html5lib=(
"html5lib" in options.deprecated_features_enabled
),
)
best_candidate = finder.find_best_candidate("pip").best_candidate
if best_candidate is None:
@@ -168,11 +165,7 @@ def pip_self_version_check(session: PipSession, options: optparse.Values) -> Non
# We cannot tell how the current pip is available in the current
# command context, so be pragmatic here and suggest the command
# that's always available. This does not accommodate spaces in
# `sys.executable` on purpose as it is not possible to do it
# correctly without knowing the user's shell. Thus,
# it won't be done until possible through the standard library.
# Do not be tempted to use the undocumented subprocess.list2cmdline.
# It is considered an internal implementation detail for a reason.
# `sys.executable`.
pip_cmd = f"{sys.executable} -m pip"
logger.warning(
"You are using pip version %s; however, version %s is "
@@ -4,28 +4,28 @@ import logging
import logging.handlers
import os
import sys
import threading
from dataclasses import dataclass
from logging import Filter
from typing import IO, Any, ClassVar, Iterator, List, Optional, TextIO, Type
from typing import IO, Any, Callable, Iterator, Optional, TextIO, Type, cast
from pip._vendor.rich.console import (
Console,
ConsoleOptions,
ConsoleRenderable,
RenderResult,
)
from pip._vendor.rich.highlighter import NullHighlighter
from pip._vendor.rich.logging import RichHandler
from pip._vendor.rich.segment import Segment
from pip._vendor.rich.style import Style
from pip._internal.exceptions import DiagnosticPipError
from pip._internal.utils._log import VERBOSE, getLogger
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX
from pip._internal.utils.misc import ensure_dir
try:
import threading
except ImportError:
import dummy_threading as threading # type: ignore
try:
from pip._vendor import colorama
# Lots of different errors can come from this, including SystemError and
# ImportError.
except Exception:
colorama = None
_log_state = threading.local()
subprocess_logger = getLogger("pip.subprocessor")
@@ -119,63 +119,78 @@ class IndentingFormatter(logging.Formatter):
return formatted
@dataclass
class IndentedRenderable:
renderable: ConsoleRenderable
indent: int
def _color_wrap(*colors: str) -> Callable[[str], str]:
def wrapped(inp: str) -> str:
return "".join(list(colors) + [inp, colorama.Style.RESET_ALL])
def __rich_console__(
self, console: Console, options: ConsoleOptions
) -> RenderResult:
segments = console.render(self.renderable, options)
lines = Segment.split_lines(segments)
for line in lines:
yield Segment(" " * self.indent)
yield from line
yield Segment("\n")
return wrapped
class RichPipStreamHandler(RichHandler):
KEYWORDS: ClassVar[Optional[List[str]]] = []
class ColorizedStreamHandler(logging.StreamHandler):
def __init__(self, stream: Optional[TextIO], no_color: bool) -> None:
super().__init__(
console=Console(file=stream, no_color=no_color, soft_wrap=True),
show_time=False,
show_level=False,
show_path=False,
highlighter=NullHighlighter(),
# Don't build up a list of colors if we don't have colorama
if colorama:
COLORS = [
# This needs to be in order from highest logging level to lowest.
(logging.ERROR, _color_wrap(colorama.Fore.RED)),
(logging.WARNING, _color_wrap(colorama.Fore.YELLOW)),
]
else:
COLORS = []
def __init__(self, stream: Optional[TextIO] = None, no_color: bool = None) -> None:
super().__init__(stream)
self._no_color = no_color
if WINDOWS and colorama:
self.stream = colorama.AnsiToWin32(self.stream)
def _using_stdout(self) -> bool:
"""
Return whether the handler is using sys.stdout.
"""
if WINDOWS and colorama:
# Then self.stream is an AnsiToWin32 object.
stream = cast(colorama.AnsiToWin32, self.stream)
return stream.wrapped is sys.stdout
return self.stream is sys.stdout
def should_color(self) -> bool:
# Don't colorize things if we do not have colorama or if told not to
if not colorama or self._no_color:
return False
real_stream = (
self.stream
if not isinstance(self.stream, colorama.AnsiToWin32)
else self.stream.wrapped
)
# Our custom override on Rich's logger, to make things work as we need them to.
def emit(self, record: logging.LogRecord) -> None:
style: Optional[Style] = None
# If the stream is a tty we should color it
if hasattr(real_stream, "isatty") and real_stream.isatty():
return True
# If we are given a diagnostic error to present, present it with indentation.
if record.msg == "[present-diagnostic] %s" and len(record.args) == 1:
diagnostic_error: DiagnosticPipError = record.args[0] # type: ignore[index]
assert isinstance(diagnostic_error, DiagnosticPipError)
# If we have an ANSI term we should color it
if os.environ.get("TERM") == "ANSI":
return True
renderable: ConsoleRenderable = IndentedRenderable(
diagnostic_error, indent=get_indentation()
)
else:
message = self.format(record)
renderable = self.render_message(record, message)
if record.levelno is not None:
if record.levelno >= logging.ERROR:
style = Style(color="red")
elif record.levelno >= logging.WARNING:
style = Style(color="yellow")
# If anything else we should not color it
return False
try:
self.console.print(renderable, overflow="ignore", crop=False, style=style)
except Exception:
self.handleError(record)
def format(self, record: logging.LogRecord) -> str:
msg = super().format(record)
if self.should_color():
for level, color in self.COLORS:
if record.levelno >= level:
msg = color(msg)
break
return msg
# The logging module says handleError() can be customized.
def handleError(self, record: logging.LogRecord) -> None:
"""Called when logging is unable to log some output."""
exc_class, exc = sys.exc_info()[:2]
# If a broken pipe occurred while calling write() or flush() on the
# stdout stream in logging's Handler.emit(), then raise our special
@@ -184,7 +199,7 @@ class RichPipStreamHandler(RichHandler):
if (
exc_class
and exc
and self.console.file is sys.stdout
and self._using_stdout()
and _is_broken_pipe_error(exc_class, exc)
):
raise BrokenStdoutLoggingError()
@@ -260,7 +275,7 @@ def setup_logging(verbosity: int, no_color: bool, user_log_file: Optional[str])
"stderr": "ext://sys.stderr",
}
handler_classes = {
"stream": "pip._internal.utils.logging.RichPipStreamHandler",
"stream": "pip._internal.utils.logging.ColorizedStreamHandler",
"file": "pip._internal.utils.logging.BetterRotatingFileHandler",
}
handlers = ["console", "console_errors", "console_subprocess"] + (
@@ -318,8 +333,8 @@ def setup_logging(verbosity: int, no_color: bool, user_log_file: Optional[str])
"console_subprocess": {
"level": level,
"class": handler_classes["stream"],
"stream": log_streams["stderr"],
"no_color": no_color,
"stream": log_streams["stderr"],
"filters": ["restrict_to_subprocess"],
"formatter": "indent",
},
@@ -32,12 +32,14 @@ from typing import (
cast,
)
from pip._vendor.pkg_resources import Distribution
from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
from pip import __version__
from pip._internal.exceptions import CommandError
from pip._internal.locations import get_major_minor_version
from pip._internal.locations import get_major_minor_version, site_packages, user_site
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.egg_link import egg_link_path_from_location
from pip._internal.utils.virtualenv import running_under_virtualenv
__all__ = [
@@ -326,6 +328,64 @@ def is_local(path: str) -> bool:
return path.startswith(normalize_path(sys.prefix))
def dist_is_local(dist: Distribution) -> bool:
"""
Return True if given Distribution object is installed locally
(i.e. within current virtualenv).
Always True if we're not in a virtualenv.
"""
return is_local(dist_location(dist))
def dist_in_usersite(dist: Distribution) -> bool:
"""
Return True if given Distribution is installed in user site.
"""
return dist_location(dist).startswith(normalize_path(user_site))
def dist_in_site_packages(dist: Distribution) -> bool:
"""
Return True if given Distribution is installed in
sysconfig.get_python_lib().
"""
return dist_location(dist).startswith(normalize_path(site_packages))
def get_distribution(req_name: str) -> Optional[Distribution]:
"""Given a requirement name, return the installed Distribution object.
This searches from *all* distributions available in the environment, to
match the behavior of ``pkg_resources.get_distribution()``.
Left for compatibility until direct pkg_resources uses are refactored out.
"""
from pip._internal.metadata import get_default_environment
from pip._internal.metadata.pkg_resources import Distribution as _Dist
dist = get_default_environment().get_distribution(req_name)
if dist is None:
return None
return cast(_Dist, dist)._dist
def dist_location(dist: Distribution) -> str:
"""
Get the site-packages location of this distribution. Generally
this is dist.location, except in the case of develop-installed
packages, where dist.location is the source code location, and we
want to know where the egg-link file is.
The returned location is normalized (in particular, with symlinks removed).
"""
egg_link = egg_link_path_from_location(dist.project_name)
if egg_link:
return normalize_path(egg_link)
return normalize_path(dist.location)
def write_output(msg: Any, *args: Any) -> None:
logger.info(msg, *args)
@@ -1,12 +1,16 @@
import functools
import logging
import re
from typing import NewType, Optional, Tuple, cast
from email.message import Message
from email.parser import FeedParser
from typing import Optional, Tuple
from pip._vendor import pkg_resources
from pip._vendor.packaging import specifiers, version
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.pkg_resources import Distribution
NormalizedExtra = NewType("NormalizedExtra", str)
from pip._internal.exceptions import NoneMetadataError
from pip._internal.utils.misc import display_path
logger = logging.getLogger(__name__)
@@ -34,6 +38,41 @@ def check_requires_python(
return python_version in requires_python_specifier
def get_metadata(dist: Distribution) -> Message:
"""
:raises NoneMetadataError: if the distribution reports `has_metadata()`
True but `get_metadata()` returns None.
"""
metadata_name = "METADATA"
if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata(
metadata_name
):
metadata = dist.get_metadata(metadata_name)
elif dist.has_metadata("PKG-INFO"):
metadata_name = "PKG-INFO"
metadata = dist.get_metadata(metadata_name)
else:
logger.warning("No metadata found in %s", display_path(dist.location))
metadata = ""
if metadata is None:
raise NoneMetadataError(dist, metadata_name)
feed_parser = FeedParser()
# The following line errors out if with a "NoneType" TypeError if
# passed metadata=None.
feed_parser.feed(metadata)
return feed_parser.close()
def get_installer(dist: Distribution) -> str:
if dist.has_metadata("INSTALLER"):
for line in dist.get_metadata_lines("INSTALLER"):
if line.strip():
return line.strip()
return ""
@functools.lru_cache(maxsize=512)
def get_requirement(req_string: str) -> Requirement:
"""Construct a packaging.Requirement object with caching"""
@@ -43,15 +82,3 @@ def get_requirement(req_string: str) -> Requirement:
# minimize repeated parsing of the same string to construct equivalent
# Requirement objects.
return Requirement(req_string)
def safe_extra(extra: str) -> NormalizedExtra:
"""Convert an arbitrary string to a standard 'extra' name
Any runs of non-alphanumeric characters are replaced with a single '_',
and the result is always lowercased.
This function is duplicated from ``pkg_resources``. Note that this is not
the same to either ``canonicalize_name`` or ``_egg_link_name``.
"""
return cast(NormalizedExtra, re.sub("[^A-Za-z0-9.-]+", "_", extra).lower())
@@ -1,49 +1,21 @@
import sys
import textwrap
from typing import List, Optional, Sequence
# Shim to wrap setup.py invocation with setuptools
# Note that __file__ is handled via two {!r} *and* %r, to ensure that paths on
# Windows are correctly handled (it should be "C:\\Users" not "C:\Users").
_SETUPTOOLS_SHIM = textwrap.dedent(
"""
exec(compile('''
# This is <pip-setuptools-caller> -- a caller that pip uses to run setup.py
#
# - It imports setuptools before invoking setup.py, to enable projects that directly
# import from `distutils.core` to work with newer packaging standards.
# - It provides a clear error message when setuptools is not installed.
# - It sets `sys.argv[0]` to the underlying `setup.py`, when invoking `setup.py` so
# setuptools doesn't think the script is `-c`. This avoids the following warning:
# manifest_maker: standard file '-c' not found".
# - It generates a shim setup.py, for handling setup.cfg-only projects.
import os, sys, tokenize
try:
import setuptools
except ImportError as error:
print(
"ERROR: Can not execute `setup.py` since setuptools is not available in "
"the build environment.",
file=sys.stderr,
)
sys.exit(1)
__file__ = %r
sys.argv[0] = __file__
if os.path.exists(__file__):
filename = __file__
with tokenize.open(__file__) as f:
setup_py_code = f.read()
else:
filename = "<auto-generated setuptools caller>"
setup_py_code = "from setuptools import setup; setup()"
exec(compile(setup_py_code, filename, "exec"))
''' % ({!r},), "<pip-setuptools-caller>", "exec"))
"""
).rstrip()
#
# We set sys.argv[0] to the path to the underlying setup.py file so
# setuptools / distutils don't take the path to the setup.py to be "-c" when
# invoking via the shim. This avoids e.g. the following manifest_maker
# warning: "warning: manifest_maker: standard file '-c' not found".
_SETUPTOOLS_SHIM = (
"import io, os, sys, setuptools, tokenize; sys.argv[0] = {0!r}; __file__={0!r};"
"f = getattr(tokenize, 'open', open)(__file__) "
"if os.path.exists(__file__) "
"else io.StringIO('from setuptools import setup; setup()');"
"code = f.read().replace('\\r\\n', '\\n');"
"f.close();"
"exec(compile(code, __file__, 'exec'))"
)
def make_setuptools_shim_args(
@@ -13,8 +13,6 @@ from typing import (
Union,
)
from pip._vendor.rich.markup import escape
from pip._internal.cli.spinners import SpinnerInterface, open_spinner
from pip._internal.exceptions import InstallationSubprocessError
from pip._internal.utils.logging import VERBOSE, subprocess_logger
@@ -29,6 +27,9 @@ if TYPE_CHECKING:
CommandArgs = List[Union[str, HiddenText]]
LOG_DIVIDER = "----------------------------------------"
def make_command(*args: Union[str, HiddenText, CommandArgs]) -> CommandArgs:
"""
Create a CommandArgs object.
@@ -68,19 +69,53 @@ def reveal_command_args(args: Union[List[str], CommandArgs]) -> List[str]:
return [arg.secret if isinstance(arg, HiddenText) else arg for arg in args]
def make_subprocess_output_error(
cmd_args: Union[List[str], CommandArgs],
cwd: Optional[str],
lines: List[str],
exit_status: int,
) -> str:
"""
Create and return the error message to use to log a subprocess error
with command output.
:param lines: A list of lines, each ending with a newline.
"""
command = format_command_args(cmd_args)
# We know the joined output value ends in a newline.
output = "".join(lines)
msg = (
# Use a unicode string to avoid "UnicodeEncodeError: 'ascii'
# codec can't encode character ..." in Python 2 when a format
# argument (e.g. `output`) has a non-ascii character.
"Command errored out with exit status {exit_status}:\n"
" command: {command_display}\n"
" cwd: {cwd_display}\n"
"Complete output ({line_count} lines):\n{output}{divider}"
).format(
exit_status=exit_status,
command_display=command,
cwd_display=cwd,
line_count=len(lines),
output=output,
divider=LOG_DIVIDER,
)
return msg
def call_subprocess(
cmd: Union[List[str], CommandArgs],
show_stdout: bool = False,
cwd: Optional[str] = None,
on_returncode: 'Literal["raise", "warn", "ignore"]' = "raise",
extra_ok_returncodes: Optional[Iterable[int]] = None,
command_desc: Optional[str] = None,
extra_environ: Optional[Mapping[str, Any]] = None,
unset_environ: Optional[Iterable[str]] = None,
spinner: Optional[SpinnerInterface] = None,
log_failed_cmd: Optional[bool] = True,
stdout_only: Optional[bool] = False,
*,
command_desc: str,
) -> str:
"""
Args:
@@ -131,6 +166,9 @@ def call_subprocess(
# and we have a spinner.
use_spinner = not showing_subprocess and spinner is not None
if command_desc is None:
command_desc = format_command_args(cmd)
log_subprocess("Running command %s", command_desc)
env = os.environ.copy()
if extra_environ:
@@ -203,25 +241,17 @@ def call_subprocess(
spinner.finish("done")
if proc_had_error:
if on_returncode == "raise":
error = InstallationSubprocessError(
command_description=command_desc,
exit_code=proc.returncode,
output_lines=all_output if not showing_subprocess else None,
)
if log_failed_cmd:
subprocess_logger.error("[present-diagnostic] %s", error)
subprocess_logger.verbose(
"[bold magenta]full command[/]: [blue]%s[/]",
escape(format_command_args(cmd)),
extra={"markup": True},
if not showing_subprocess and log_failed_cmd:
# Then the subprocess streams haven't been logged to the
# console yet.
msg = make_subprocess_output_error(
cmd_args=cmd,
cwd=cwd,
lines=all_output,
exit_status=proc.returncode,
)
subprocess_logger.verbose(
"[bold magenta]cwd[/]: %s",
escape(cwd or "[inherit]"),
extra={"markup": True},
)
raise error
subprocess_logger.error(msg)
raise InstallationSubprocessError(proc.returncode, command_desc)
elif on_returncode == "warn":
subprocess_logger.warning(
'Command "%s" had error code %s in %s',
@@ -251,7 +281,6 @@ def runner_with_spinner_message(message: str) -> Callable[..., None]:
with open_spinner(message) as spinner:
call_subprocess(
cmd,
command_desc=message,
cwd=cwd,
extra_environ=extra_environ,
spinner=spinner,
@@ -4,12 +4,14 @@
import logging
from email.message import Message
from email.parser import Parser
from typing import Tuple
from typing import Dict, Tuple
from zipfile import BadZipFile, ZipFile
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.pkg_resources import DistInfoDistribution, Distribution
from pip._internal.exceptions import UnsupportedWheel
from pip._internal.utils.pkg_resources import DictMetadata
VERSION_COMPATIBLE = (1, 0)
@@ -17,6 +19,50 @@ VERSION_COMPATIBLE = (1, 0)
logger = logging.getLogger(__name__)
class WheelMetadata(DictMetadata):
"""Metadata provider that maps metadata decoding exceptions to our
internal exception type.
"""
def __init__(self, metadata: Dict[str, bytes], wheel_name: str) -> None:
super().__init__(metadata)
self._wheel_name = wheel_name
def get_metadata(self, name: str) -> str:
try:
return super().get_metadata(name)
except UnicodeDecodeError as e:
# Augment the default error with the origin of the file.
raise UnsupportedWheel(
f"Error decoding metadata for {self._wheel_name}: {e}"
)
def pkg_resources_distribution_for_wheel(
wheel_zip: ZipFile, name: str, location: str
) -> Distribution:
"""Get a pkg_resources distribution given a wheel.
:raises UnsupportedWheel: on any errors
"""
info_dir, _ = parse_wheel(wheel_zip, name)
metadata_files = [p for p in wheel_zip.namelist() if p.startswith(f"{info_dir}/")]
metadata_text: Dict[str, bytes] = {}
for path in metadata_files:
_, metadata_name = path.split("/", 1)
try:
metadata_text[metadata_name] = read_wheel_metadata_file(wheel_zip, path)
except UnsupportedWheel as e:
raise UnsupportedWheel("{} has an invalid wheel, {}".format(name, str(e)))
metadata = WheelMetadata(metadata_text, location)
return DistInfoDistribution(location=location, metadata=metadata, project_name=name)
def parse_wheel(wheel_zip: ZipFile, name: str) -> Tuple[str, Message]:
"""Extract information from the provided wheel, ensuring it meets basic
standards.
@@ -33,9 +33,7 @@ class Bazaar(VersionControl):
def get_base_rev_args(rev: str) -> List[str]:
return ["-r", rev]
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
rev_display = rev_options.to_display()
logger.info(
"Checking out %s%s to %s",
@@ -43,13 +41,7 @@ class Bazaar(VersionControl):
rev_display,
display_path(dest),
)
if verbosity <= 0:
flag = "--quiet"
elif verbosity == 1:
flag = ""
else:
flag = f"-{'v'*verbosity}"
cmd_args = make_command("branch", flag, rev_options.to_args(), url, dest)
cmd_args = make_command("branch", "-q", rev_options.to_args(), url, dest)
self.run_command(cmd_args)
def switch(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
@@ -91,12 +91,7 @@ class Git(VersionControl):
return not is_tag_or_branch
def get_git_version(self) -> Tuple[int, ...]:
version = self.run_command(
["version"],
command_desc="git version",
show_stdout=False,
stdout_only=True,
)
version = self.run_command(["version"], show_stdout=False, stdout_only=True)
match = GIT_VERSION_REGEX.match(version)
if not match:
logger.warning("Can't parse git version: %s", version)
@@ -258,17 +253,9 @@ class Git(VersionControl):
return cls.get_revision(dest) == name
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
rev_display = rev_options.to_display()
logger.info("Cloning %s%s to %s", url, rev_display, display_path(dest))
if verbosity <= 0:
flags: Tuple[str, ...] = ("--quiet",)
elif verbosity == 1:
flags = ()
else:
flags = ("--verbose", "--progress")
if self.get_git_version() >= (2, 17):
# Git added support for partial clone in 2.17
# https://git-scm.com/docs/partial-clone
@@ -277,13 +264,13 @@ class Git(VersionControl):
make_command(
"clone",
"--filter=blob:none",
*flags,
"-q",
url,
dest,
)
)
else:
self.run_command(make_command("clone", *flags, url, dest))
self.run_command(make_command("clone", "-q", url, dest))
if rev_options.rev:
# Then a specific revision was requested.
@@ -1,7 +1,7 @@
import configparser
import logging
import os
from typing import List, Optional, Tuple
from typing import List, Optional
from pip._internal.exceptions import BadCommand, InstallationError
from pip._internal.utils.misc import HiddenText, display_path
@@ -33,9 +33,7 @@ class Mercurial(VersionControl):
def get_base_rev_args(rev: str) -> List[str]:
return [rev]
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
rev_display = rev_options.to_display()
logger.info(
"Cloning hg %s%s to %s",
@@ -43,17 +41,9 @@ class Mercurial(VersionControl):
rev_display,
display_path(dest),
)
if verbosity <= 0:
flags: Tuple[str, ...] = ("--quiet",)
elif verbosity == 1:
flags = ()
elif verbosity == 2:
flags = ("--verbose",)
else:
flags = ("--verbose", "--debug")
self.run_command(make_command("clone", "--noupdate", *flags, url, dest))
self.run_command(make_command("clone", "--noupdate", "-q", url, dest))
self.run_command(
make_command("update", *flags, rev_options.to_args()),
make_command("update", "-q", rev_options.to_args()),
cwd=dest,
)
@@ -277,9 +277,7 @@ class Subversion(VersionControl):
return []
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
rev_display = rev_options.to_display()
logger.info(
"Checking out %s%s to %s",
@@ -287,13 +285,9 @@ class Subversion(VersionControl):
rev_display,
display_path(dest),
)
if verbosity <= 0:
flag = "--quiet"
else:
flag = ""
cmd_args = make_command(
"checkout",
flag,
"-q",
self.get_remote_call_options(),
rev_options.to_args(),
url,
@@ -31,12 +31,7 @@ from pip._internal.utils.misc import (
is_installable_dir,
rmtree,
)
from pip._internal.utils.subprocess import (
CommandArgs,
call_subprocess,
format_command_args,
make_command,
)
from pip._internal.utils.subprocess import CommandArgs, call_subprocess, make_command
from pip._internal.utils.urls import get_url_scheme
if TYPE_CHECKING:
@@ -463,9 +458,7 @@ class VersionControl:
"""
return cls.normalize_url(url1) == cls.normalize_url(url2)
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
) -> None:
def fetch_new(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
"""
Fetch a revision from a repository, in the case that this is the
first fetch from the repository.
@@ -473,7 +466,6 @@ class VersionControl:
Args:
dest: the directory to fetch the repository to.
rev_options: a RevOptions object.
verbosity: verbosity level.
"""
raise NotImplementedError
@@ -506,19 +498,18 @@ class VersionControl:
"""
raise NotImplementedError
def obtain(self, dest: str, url: HiddenText, verbosity: int) -> None:
def obtain(self, dest: str, url: HiddenText) -> None:
"""
Install or update in editable mode the package represented by this
VersionControl object.
:param dest: the repository directory in which to install or update.
:param url: the repository URL starting with a vcs prefix.
:param verbosity: verbosity level.
"""
url, rev_options = self.get_url_rev_options(url)
if not os.path.exists(dest):
self.fetch_new(dest, url, rev_options, verbosity=verbosity)
self.fetch_new(dest, url, rev_options)
return
rev_display = rev_options.to_display()
@@ -574,14 +565,14 @@ class VersionControl:
if response == "w":
logger.warning("Deleting %s", display_path(dest))
rmtree(dest)
self.fetch_new(dest, url, rev_options, verbosity=verbosity)
self.fetch_new(dest, url, rev_options)
return
if response == "b":
dest_dir = backup_dir(dest)
logger.warning("Backing up %s to %s", display_path(dest), dest_dir)
shutil.move(dest, dest_dir)
self.fetch_new(dest, url, rev_options, verbosity=verbosity)
self.fetch_new(dest, url, rev_options)
return
# Do nothing if the response is "i".
@@ -595,17 +586,16 @@ class VersionControl:
)
self.switch(dest, url, rev_options)
def unpack(self, location: str, url: HiddenText, verbosity: int) -> None:
def unpack(self, location: str, url: HiddenText) -> None:
"""
Clean up current location and download the url repository
(and vcs infos) into location
:param url: the repository URL starting with a vcs prefix.
:param verbosity: verbosity level.
"""
if os.path.exists(location):
rmtree(location)
self.obtain(location, url=url, verbosity=verbosity)
self.obtain(location, url=url)
@classmethod
def get_remote_url(cls, location: str) -> str:
@@ -644,8 +634,6 @@ class VersionControl:
command name, and checks that the VCS is available
"""
cmd = make_command(cls.name, *cmd)
if command_desc is None:
command_desc = format_command_args(cmd)
try:
return call_subprocess(
cmd,
@@ -310,9 +310,7 @@ def _clean_one_legacy(req: InstallRequirement, global_options: List[str]) -> boo
logger.info("Running setup.py clean for %s", req.name)
try:
call_subprocess(
clean_args, command_desc="python setup.py clean", cwd=req.source_dir
)
call_subprocess(clean_args, cwd=req.source_dir)
return True
except Exception:
logger.error("Failed cleaning build dir for %s", req.name)
@@ -1,18 +1,11 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
"""CacheControl import Interface.
Make it easy to import from cachecontrol without long namespaces.
"""
__author__ = "Eric Larson"
__email__ = "eric@ionrock.org"
__version__ = "0.12.10"
__version__ = "0.12.6"
from .wrapper import CacheControl
from .adapter import CacheControlAdapter
from .controller import CacheController
import logging
logging.getLogger(__name__).addHandler(logging.NullHandler())
@@ -1,7 +1,3 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
import logging
from pip._vendor import requests
@@ -1,20 +1,16 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
import types
import functools
import zlib
from pip._vendor.requests.adapters import HTTPAdapter
from .controller import CacheController, PERMANENT_REDIRECT_STATUSES
from .controller import CacheController
from .cache import DictCache
from .filewrapper import CallbackFileWrapper
class CacheControlAdapter(HTTPAdapter):
invalidating_methods = {"PUT", "PATCH", "DELETE"}
invalidating_methods = {"PUT", "DELETE"}
def __init__(
self,
@@ -97,7 +93,7 @@ class CacheControlAdapter(HTTPAdapter):
response = cached_response
# We always cache the 301 responses
elif int(response.status) in PERMANENT_REDIRECT_STATUSES:
elif response.status == 301:
self.controller.cache_response(request, response)
else:
# Wrap the response file with a wrapper that will cache the
@@ -1,7 +1,3 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
"""
The cache object API for implementing caches. The default is a thread
safe in-memory dictionary.
@@ -14,7 +10,7 @@ class BaseCache(object):
def get(self, key):
raise NotImplementedError()
def set(self, key, value, expires=None):
def set(self, key, value):
raise NotImplementedError()
def delete(self, key):
@@ -33,7 +29,7 @@ class DictCache(BaseCache):
def get(self, key):
return self.data.get(key, None)
def set(self, key, value, expires=None):
def set(self, key, value):
with self.lock:
self.data.update({key: value})
@@ -1,6 +1,2 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from .file_cache import FileCache # noqa
from .redis_cache import RedisCache # noqa
@@ -1,7 +1,3 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
import hashlib
import os
from textwrap import dedent
@@ -118,7 +114,7 @@ class FileCache(BaseCache):
except FileNotFoundError:
return None
def set(self, key, value, expires=None):
def set(self, key, value):
name = self._fn(key)
# Make sure the directory exists
@@ -1,7 +1,3 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import division
from datetime import datetime
@@ -1,7 +1,3 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
try:
from urllib.parse import urljoin
except ImportError:
@@ -13,6 +9,7 @@ try:
except ImportError:
import pickle
# Handle the case where the requests module has been patched to not have
# urllib3 bundled as part of its source.
try:
@@ -1,7 +1,3 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
"""
The httplib2 algorithms ported for use with requests.
"""
@@ -21,8 +17,6 @@ logger = logging.getLogger(__name__)
URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
PERMANENT_REDIRECT_STATUSES = (301, 308)
def parse_uri(uri):
"""Parses a URI using the regex given in Appendix B of RFC 3986.
@@ -43,7 +37,7 @@ class CacheController(object):
self.cache = DictCache() if cache is None else cache
self.cache_etags = cache_etags
self.serializer = serializer or Serializer()
self.cacheable_status_codes = status_codes or (200, 203, 300, 301, 308)
self.cacheable_status_codes = status_codes or (200, 203, 300, 301)
@classmethod
def _urlnorm(cls, uri):
@@ -153,18 +147,17 @@ class CacheController(object):
logger.warning("Cache entry deserialization failed, entry ignored")
return False
# If we have a cached permanent redirect, return it immediately. We
# don't need to test our response for other headers b/c it is
# If we have a cached 301, return it immediately. We don't
# need to test our response for other headers b/c it is
# intrinsically "cacheable" as it is Permanent.
#
# See:
# https://tools.ietf.org/html/rfc7231#section-6.4.2
#
# Client can try to refresh the value by repeating the request
# with cache busting headers as usual (ie no-cache).
if int(resp.status) in PERMANENT_REDIRECT_STATUSES:
if resp.status == 301:
msg = (
"Returning cached permanent redirect response "
'Returning cached "301 Moved Permanently" response '
"(ignoring date and etag information)"
)
logger.debug(msg)
@@ -268,11 +261,6 @@ class CacheController(object):
response_headers = CaseInsensitiveDict(response.headers)
if "date" in response_headers:
date = calendar.timegm(parsedate_tz(response_headers["date"]))
else:
date = 0
# If we've been given a body, our response has a Content-Length, that
# Content-Length is valid then we can check to see if the body we've
# been given matches the expected size, and if it doesn't we'll just
@@ -316,62 +304,35 @@ class CacheController(object):
# If we've been given an etag, then keep the response
if self.cache_etags and "etag" in response_headers:
expires_time = 0
if response_headers.get("expires"):
expires = parsedate_tz(response_headers["expires"])
if expires is not None:
expires_time = calendar.timegm(expires) - date
expires_time = max(expires_time, 14 * 86400)
logger.debug("etag object cached for {0} seconds".format(expires_time))
logger.debug("Caching due to etag")
self.cache.set(
cache_url,
self.serializer.dumps(request, response, body),
expires=expires_time,
cache_url, self.serializer.dumps(request, response, body=body)
)
# Add to the cache any permanent redirects. We do this before looking
# that the Date headers.
elif int(response.status) in PERMANENT_REDIRECT_STATUSES:
logger.debug("Caching permanent redirect")
self.cache.set(cache_url, self.serializer.dumps(request, response, b""))
# Add to the cache any 301s. We do this before looking that
# the Date headers.
elif response.status == 301:
logger.debug("Caching permanant redirect")
self.cache.set(cache_url, self.serializer.dumps(request, response))
# Add to the cache if the response headers demand it. If there
# is no date header then we can't do anything about expiring
# the cache.
elif "date" in response_headers:
date = calendar.timegm(parsedate_tz(response_headers["date"]))
# cache when there is a max-age > 0
if "max-age" in cc and cc["max-age"] > 0:
logger.debug("Caching b/c date exists and max-age > 0")
expires_time = cc["max-age"]
self.cache.set(
cache_url,
self.serializer.dumps(request, response, body),
expires=expires_time,
cache_url, self.serializer.dumps(request, response, body=body)
)
# If the request can expire, it means we should cache it
# in the meantime.
elif "expires" in response_headers:
if response_headers["expires"]:
expires = parsedate_tz(response_headers["expires"])
if expires is not None:
expires_time = calendar.timegm(expires) - date
else:
expires_time = None
logger.debug(
"Caching b/c of expires header. expires in {0} seconds".format(
expires_time
)
)
logger.debug("Caching b/c of expires header")
self.cache.set(
cache_url,
self.serializer.dumps(request, response, body=body),
expires=expires_time,
cache_url, self.serializer.dumps(request, response, body=body)
)
def update_cached_response(self, request, response):
@@ -1,9 +1,4 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from tempfile import NamedTemporaryFile
import mmap
from io import BytesIO
class CallbackFileWrapper(object):
@@ -16,17 +11,10 @@ class CallbackFileWrapper(object):
This class uses members with a double underscore (__) leading prefix so as
not to accidentally shadow an attribute.
The data is stored in a temporary file until it is all available. As long
as the temporary files directory is disk-based (sometimes it's a
memory-backed-``tmpfs`` on Linux), data will be unloaded to disk if memory
pressure is high. For small files the disk usually won't be used at all,
it'll all be in the filesystem memory cache, so there should be no
performance impact.
"""
def __init__(self, fp, callback):
self.__buf = NamedTemporaryFile("rb+", delete=True)
self.__buf = BytesIO()
self.__fp = fp
self.__callback = callback
@@ -61,19 +49,7 @@ class CallbackFileWrapper(object):
def _close(self):
if self.__callback:
if self.__buf.tell() == 0:
# Empty file:
result = b""
else:
# Return the data without actually loading it into memory,
# relying on Python's buffer API and mmap(). mmap() just gives
# a view directly into the filesystem's memory cache, so it
# doesn't result in duplicate memory use.
self.__buf.seek(0, 0)
result = memoryview(
mmap.mmap(self.__buf.fileno(), 0, access=mmap.ACCESS_READ)
)
self.__callback(result)
self.__callback(self.__buf.getvalue())
# We assign this to None here, because otherwise we can get into
# really tricky problems where the CPython interpreter dead locks
@@ -82,16 +58,9 @@ class CallbackFileWrapper(object):
# and allows the garbage collector to do it's thing normally.
self.__callback = None
# Closing the temporary file releases memory and frees disk space.
# Important when caching big files.
self.__buf.close()
def read(self, amt=None):
data = self.__fp.read(amt)
if data:
# We may be dealing with b'', a sign that things are over:
# it's passed e.g. after we've already closed self.__buf.
self.__buf.write(data)
self.__buf.write(data)
if self.__is_fp_closed():
self._close()
@@ -1,7 +1,3 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
import calendar
import time
@@ -1,7 +1,3 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
import base64
import io
import json
@@ -21,18 +17,24 @@ def _b64_decode_str(s):
return _b64_decode_bytes(s).decode("utf8")
_default_body_read = object()
class Serializer(object):
def dumps(self, request, response, body=None):
response_headers = CaseInsensitiveDict(response.headers)
if body is None:
# When a body isn't passed in, we'll read the response. We
# also update the response with a new file handler to be
# sure it acts as though it was never read.
body = response.read(decode_content=False)
# NOTE: 99% sure this is dead code. I'm only leaving it
# here b/c I don't have a test yet to prove
# it. Basically, before using
# `cachecontrol.filewrapper.CallbackFileWrapper`,
# this made an effort to reset the file handle. The
# `CallbackFileWrapper` short circuits this code by
# setting the body as the content is consumed, the
# result being a `body` argument is *always* passed
# into cache_response, and in turn,
# `Serializer.dump`.
response._fp = io.BytesIO(body)
# NOTE: This is all a bit weird, but it's really important that on
@@ -1,7 +1,3 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from .adapter import CacheControlAdapter
from .cache import DictCache
@@ -1,3 +1,3 @@
from .core import contents, where
__version__ = "2021.10.08"
__version__ = "2021.05.30"
@@ -4255,108 +4255,3 @@ qJZ9ZPskWkoDbGs4xugDQ5r3V7mzKWmTOPQD8rv7gmsHINFSH5pkAnuYZttcTVoP
0ISVoDwUQwbKytu4QTbaakRnh6+v40URFWkIsr4WOZckbxJF0WddCajJFdr60qZf
E2Efv4WstK2tBZQIgx51F9NxO5NQI1mg7TyRVJ12AMXDuDjb
-----END CERTIFICATE-----
# Issuer: CN=TunTrust Root CA O=Agence Nationale de Certification Electronique
# Subject: CN=TunTrust Root CA O=Agence Nationale de Certification Electronique
# Label: "TunTrust Root CA"
# Serial: 108534058042236574382096126452369648152337120275
# MD5 Fingerprint: 85:13:b9:90:5b:36:5c:b6:5e:b8:5a:f8:e0:31:57:b4
# SHA1 Fingerprint: cf:e9:70:84:0f:e0:73:0f:9d:f6:0c:7f:2c:4b:ee:20:46:34:9c:bb
# SHA256 Fingerprint: 2e:44:10:2a:b5:8c:b8:54:19:45:1c:8e:19:d9:ac:f3:66:2c:af:bc:61:4b:6a:53:96:0a:30:f7:d0:e2:eb:41
-----BEGIN CERTIFICATE-----
MIIFszCCA5ugAwIBAgIUEwLV4kBMkkaGFmddtLu7sms+/BMwDQYJKoZIhvcNAQEL
BQAwYTELMAkGA1UEBhMCVE4xNzA1BgNVBAoMLkFnZW5jZSBOYXRpb25hbGUgZGUg
Q2VydGlmaWNhdGlvbiBFbGVjdHJvbmlxdWUxGTAXBgNVBAMMEFR1blRydXN0IFJv
b3QgQ0EwHhcNMTkwNDI2MDg1NzU2WhcNNDQwNDI2MDg1NzU2WjBhMQswCQYDVQQG
EwJUTjE3MDUGA1UECgwuQWdlbmNlIE5hdGlvbmFsZSBkZSBDZXJ0aWZpY2F0aW9u
IEVsZWN0cm9uaXF1ZTEZMBcGA1UEAwwQVHVuVHJ1c3QgUm9vdCBDQTCCAiIwDQYJ
KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMPN0/y9BFPdDCA61YguBUtB9YOCfvdZ
n56eY+hz2vYGqU8ftPkLHzmMmiDQfgbU7DTZhrx1W4eI8NLZ1KMKsmwb60ksPqxd
2JQDoOw05TDENX37Jk0bbjBU2PWARZw5rZzJJQRNmpA+TkBuimvNKWfGzC3gdOgF
VwpIUPp6Q9p+7FuaDmJ2/uqdHYVy7BG7NegfJ7/Boce7SBbdVtfMTqDhuazb1YMZ
GoXRlJfXyqNlC/M4+QKu3fZnz8k/9YosRxqZbwUN/dAdgjH8KcwAWJeRTIAAHDOF
li/LQcKLEITDCSSJH7UP2dl3RxiSlGBcx5kDPP73lad9UKGAwqmDrViWVSHbhlnU
r8a83YFuB9tgYv7sEG7aaAH0gxupPqJbI9dkxt/con3YS7qC0lH4Zr8GRuR5KiY2
eY8fTpkdso8MDhz/yV3A/ZAQprE38806JG60hZC/gLkMjNWb1sjxVj8agIl6qeIb
MlEsPvLfe/ZdeikZjuXIvTZxi11Mwh0/rViizz1wTaZQmCXcI/m4WEEIcb9PuISg
jwBUFfyRbVinljvrS5YnzWuioYasDXxU5mZMZl+QviGaAkYt5IPCgLnPSz7ofzwB
7I9ezX/SKEIBlYrilz0QIX32nRzFNKHsLA4KUiwSVXAkPcvCFDVDXSdOvsC9qnyW
5/yeYa1E0wCXAgMBAAGjYzBhMB0GA1UdDgQWBBQGmpsfU33x9aTI04Y+oXNZtPdE
ITAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFAaamx9TffH1pMjThj6hc1m0
90QhMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAqgVutt0Vyb+z
xiD2BkewhpMl0425yAA/l/VSJ4hxyXT968pk21vvHl26v9Hr7lxpuhbI87mP0zYu
QEkHDVneixCwSQXi/5E/S7fdAo74gShczNxtr18UnH1YeA32gAm56Q6XKRm4t+v4
FstVEuTGfbvE7Pi1HE4+Z7/FXxttbUcoqgRYYdZ2vyJ/0Adqp2RT8JeNnYA/u8EH
22Wv5psymsNUk8QcCMNE+3tjEUPRahphanltkE8pjkcFwRJpadbGNjHh/PqAulxP
xOu3Mqz4dWEX1xAZufHSCe96Qp1bWgvUxpVOKs7/B9dPfhgGiPEZtdmYu65xxBzn
dFlY7wyJz4sfdZMaBBSSSFCp61cpABbjNhzI+L/wM9VBD8TMPN3pM0MBkRArHtG5
Xc0yGYuPjCB31yLEQtyEFpslbei0VXF/sHyz03FJuc9SpAQ/3D2gu68zngowYI7b
nV2UqL1g52KAdoGDDIzMMEZJ4gzSqK/rYXHv5yJiqfdcZGyfFoxnNidF9Ql7v/YQ
CvGwjVRDjAS6oz/v4jXH+XTgbzRB0L9zZVcg+ZtnemZoJE6AZb0QmQZZ8mWvuMZH
u/2QeItBcy6vVR/cO5JyboTT0GFMDcx2V+IthSIVNg3rAZ3r2OvEhJn7wAzMMujj
d9qDRIueVSjAi1jTkD5OGwDxFa2DK5o=
-----END CERTIFICATE-----
# Issuer: CN=HARICA TLS RSA Root CA 2021 O=Hellenic Academic and Research Institutions CA
# Subject: CN=HARICA TLS RSA Root CA 2021 O=Hellenic Academic and Research Institutions CA
# Label: "HARICA TLS RSA Root CA 2021"
# Serial: 76817823531813593706434026085292783742
# MD5 Fingerprint: 65:47:9b:58:86:dd:2c:f0:fc:a2:84:1f:1e:96:c4:91
# SHA1 Fingerprint: 02:2d:05:82:fa:88:ce:14:0c:06:79:de:7f:14:10:e9:45:d7:a5:6d
# SHA256 Fingerprint: d9:5d:0e:8e:da:79:52:5b:f9:be:b1:1b:14:d2:10:0d:32:94:98:5f:0c:62:d9:fa:bd:9c:d9:99:ec:cb:7b:1d
-----BEGIN CERTIFICATE-----
MIIFpDCCA4ygAwIBAgIQOcqTHO9D88aOk8f0ZIk4fjANBgkqhkiG9w0BAQsFADBs
MQswCQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl
c2VhcmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBSU0Eg
Um9vdCBDQSAyMDIxMB4XDTIxMDIxOTEwNTUzOFoXDTQ1MDIxMzEwNTUzN1owbDEL
MAkGA1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNl
YXJjaCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgUlNBIFJv
b3QgQ0EgMjAyMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAIvC569l
mwVnlskNJLnQDmT8zuIkGCyEf3dRywQRNrhe7Wlxp57kJQmXZ8FHws+RFjZiPTgE
4VGC/6zStGndLuwRo0Xua2s7TL+MjaQenRG56Tj5eg4MmOIjHdFOY9TnuEFE+2uv
a9of08WRiFukiZLRgeaMOVig1mlDqa2YUlhu2wr7a89o+uOkXjpFc5gH6l8Cct4M
pbOfrqkdtx2z/IpZ525yZa31MJQjB/OCFks1mJxTuy/K5FrZx40d/JiZ+yykgmvw
Kh+OC19xXFyuQnspiYHLA6OZyoieC0AJQTPb5lh6/a6ZcMBaD9YThnEvdmn8kN3b
LW7R8pv1GmuebxWMevBLKKAiOIAkbDakO/IwkfN4E8/BPzWr8R0RI7VDIp4BkrcY
AuUR0YLbFQDMYTfBKnya4dC6s1BG7oKsnTH4+yPiAwBIcKMJJnkVU2DzOFytOOqB
AGMUuTNe3QvboEUHGjMJ+E20pwKmafTCWQWIZYVWrkvL4N48fS0ayOn7H6NhStYq
E613TBoYm5EPWNgGVMWX+Ko/IIqmhaZ39qb8HOLubpQzKoNQhArlT4b4UEV4AIHr
W2jjJo3Me1xR9BQsQL4aYB16cmEdH2MtiKrOokWQCPxrvrNQKlr9qEgYRtaQQJKQ
CoReaDH46+0N0x3GfZkYVVYnZS6NRcUk7M7jAgMBAAGjQjBAMA8GA1UdEwEB/wQF
MAMBAf8wHQYDVR0OBBYEFApII6ZgpJIKM+qTW8VX6iVNvRLuMA4GA1UdDwEB/wQE
AwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAPpBIqm5iFSVmewzVjIuJndftTgfvnNAU
X15QvWiWkKQUEapobQk1OUAJ2vQJLDSle1mESSmXdMgHHkdt8s4cUCbjnj1AUz/3
f5Z2EMVGpdAgS1D0NTsY9FVqQRtHBmg8uwkIYtlfVUKqrFOFrJVWNlar5AWMxaja
H6NpvVMPxP/cyuN+8kyIhkdGGvMA9YCRotxDQpSbIPDRzbLrLFPCU3hKTwSUQZqP
JzLB5UkZv/HywouoCjkxKLR9YjYsTewfM7Z+d21+UPCfDtcRj88YxeMn/ibvBZ3P
zzfF0HvaO7AWhAw6k9a+F9sPPg4ZeAnHqQJyIkv3N3a6dcSFA1pj1bF1BcK5vZSt
jBWZp5N99sXzqnTPBIWUmAD04vnKJGW/4GKvyMX6ssmeVkjaef2WdhW+o45WxLM0
/L5H9MG0qPzVMIho7suuyWPEdr6sOBjhXlzPrjoiUevRi7PzKzMHVIf6tLITe7pT
BGIBnfHAT+7hOtSLIBD6Alfm78ELt5BGnBkpjNxvoEppaZS3JGWg/6w/zgH7IS79
aPib8qXPMThcFarmlwDB31qlpzmq6YR/PFGoOtmUW4y/Twhx5duoXNTSpv4Ao8YW
xw/ogM4cKGR0GQjTQuPOAF1/sdwTsOEFy9EgqoZ0njnnkf3/W9b3raYvAwtt41dU
63ZTGI0RmLo=
-----END CERTIFICATE-----
# Issuer: CN=HARICA TLS ECC Root CA 2021 O=Hellenic Academic and Research Institutions CA
# Subject: CN=HARICA TLS ECC Root CA 2021 O=Hellenic Academic and Research Institutions CA
# Label: "HARICA TLS ECC Root CA 2021"
# Serial: 137515985548005187474074462014555733966
# MD5 Fingerprint: ae:f7:4c:e5:66:35:d1:b7:9b:8c:22:93:74:d3:4b:b0
# SHA1 Fingerprint: bc:b0:c1:9d:e9:98:92:70:19:38:57:e9:8d:a7:b4:5d:6e:ee:01:48
# SHA256 Fingerprint: 3f:99:cc:47:4a:cf:ce:4d:fe:d5:87:94:66:5e:47:8d:15:47:73:9f:2e:78:0f:1b:b4:ca:9b:13:30:97:d4:01
-----BEGIN CERTIFICATE-----
MIICVDCCAdugAwIBAgIQZ3SdjXfYO2rbIvT/WeK/zjAKBggqhkjOPQQDAzBsMQsw
CQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJlc2Vh
cmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBFQ0MgUm9v
dCBDQSAyMDIxMB4XDTIxMDIxOTExMDExMFoXDTQ1MDIxMzExMDEwOVowbDELMAkG
A1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJj
aCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgRUNDIFJvb3Qg
Q0EgMjAyMTB2MBAGByqGSM49AgEGBSuBBAAiA2IABDgI/rGgltJ6rK9JOtDA4MM7
KKrxcm1lAEeIhPyaJmuqS7psBAqIXhfyVYf8MLA04jRYVxqEU+kw2anylnTDUR9Y
STHMmE5gEYd103KUkE+bECUqqHgtvpBBWJAVcqeht6NCMEAwDwYDVR0TAQH/BAUw
AwEB/zAdBgNVHQ4EFgQUyRtTgRL+BNUW0aq8mm+3oJUZbsowDgYDVR0PAQH/BAQD
AgGGMAoGCCqGSM49BAMDA2cAMGQCMBHervjcToiwqfAircJRQO9gcS3ujwLEXQNw
SaSS6sUUiHCm0w2wqsosQJz76YJumgIwK0eaB8bRwoF8yguWGEEbo/QwCZ61IygN
nxS2PFOiTAZpffpskcYqSUXm7LcT4Tps
-----END CERTIFICATE-----
@@ -7,7 +7,8 @@ _unicode_dots_re = re.compile('[\u002e\u3002\uff0e\uff61]')
class Codec(codecs.Codec):
def encode(self, data: str, errors: str = 'strict') -> Tuple[bytes, int]:
def encode(self, data, errors='strict'):
# type: (str, str) -> Tuple[bytes, int]
if errors != 'strict':
raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
@@ -16,7 +17,8 @@ class Codec(codecs.Codec):
return encode(data), len(data)
def decode(self, data: bytes, errors: str = 'strict') -> Tuple[str, int]:
def decode(self, data, errors='strict'):
# type: (bytes, str) -> Tuple[str, int]
if errors != 'strict':
raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
@@ -26,7 +28,8 @@ class Codec(codecs.Codec):
return decode(data), len(data)
class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
def _buffer_encode(self, data: str, errors: str, final: bool) -> Tuple[str, int]: # type: ignore
def _buffer_encode(self, data, errors, final): # type: ignore
# type: (str, str, bool) -> Tuple[str, int]
if errors != 'strict':
raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
@@ -59,7 +62,8 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
return result_str, size
class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
def _buffer_decode(self, data: str, errors: str, final: bool) -> Tuple[str, int]: # type: ignore
def _buffer_decode(self, data, errors, final): # type: ignore
# type: (str, str, bool) -> Tuple[str, int]
if errors != 'strict':
raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
@@ -99,7 +103,8 @@ class StreamReader(Codec, codecs.StreamReader):
pass
def getregentry() -> codecs.CodecInfo:
def getregentry():
# type: () -> codecs.CodecInfo
# Compatibility as a search_function for codecs.register()
return codecs.CodecInfo(
name='idna',
@@ -2,12 +2,15 @@ from .core import *
from .codec import *
from typing import Any, Union
def ToASCII(label: str) -> bytes:
def ToASCII(label):
# type: (str) -> bytes
return encode(label)
def ToUnicode(label: Union[bytes, bytearray]) -> str:
def ToUnicode(label):
# type: (Union[bytes, bytearray]) -> str
return decode(label)
def nameprep(s: Any) -> None:
def nameprep(s):
# type: (Any) -> None
raise NotImplementedError('IDNA 2008 does not utilise nameprep protocol')
+39 -27
View File
@@ -29,36 +29,43 @@ class InvalidCodepointContext(IDNAError):
pass
def _combining_class(cp: int) -> int:
def _combining_class(cp):
# type: (int) -> int
v = unicodedata.combining(chr(cp))
if v == 0:
if not unicodedata.name(chr(cp)):
raise ValueError('Unknown character in unicodedata')
return v
def _is_script(cp: str, script: str) -> bool:
def _is_script(cp, script):
# type: (str, str) -> bool
return intranges_contain(ord(cp), idnadata.scripts[script])
def _punycode(s: str) -> bytes:
def _punycode(s):
# type: (str) -> bytes
return s.encode('punycode')
def _unot(s: int) -> str:
def _unot(s):
# type: (int) -> str
return 'U+{:04X}'.format(s)
def valid_label_length(label: Union[bytes, str]) -> bool:
def valid_label_length(label):
# type: (Union[bytes, str]) -> bool
if len(label) > 63:
return False
return True
def valid_string_length(label: Union[bytes, str], trailing_dot: bool) -> bool:
def valid_string_length(label, trailing_dot):
# type: (Union[bytes, str], bool) -> bool
if len(label) > (254 if trailing_dot else 253):
return False
return True
def check_bidi(label: str, check_ltr: bool = False) -> bool:
def check_bidi(label, check_ltr=False):
# type: (str, bool) -> bool
# Bidi rules should only be applied if string contains RTL characters
bidi_label = False
for (idx, cp) in enumerate(label, 1):
@@ -117,13 +124,15 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool:
return True
def check_initial_combiner(label: str) -> bool:
def check_initial_combiner(label):
# type: (str) -> bool
if unicodedata.category(label[0])[0] == 'M':
raise IDNAError('Label begins with an illegal combining character')
return True
def check_hyphen_ok(label: str) -> bool:
def check_hyphen_ok(label):
# type: (str) -> bool
if label[2:4] == '--':
raise IDNAError('Label has disallowed hyphens in 3rd and 4th position')
if label[0] == '-' or label[-1] == '-':
@@ -131,12 +140,14 @@ def check_hyphen_ok(label: str) -> bool:
return True
def check_nfc(label: str) -> None:
def check_nfc(label):
# type: (str) -> None
if unicodedata.normalize('NFC', label) != label:
raise IDNAError('Label must be in Normalization Form C')
def valid_contextj(label: str, pos: int) -> bool:
def valid_contextj(label, pos):
# type: (str, int) -> bool
cp_value = ord(label[pos])
if cp_value == 0x200c:
@@ -179,7 +190,8 @@ def valid_contextj(label: str, pos: int) -> bool:
return False
def valid_contexto(label: str, pos: int, exception: bool = False) -> bool:
def valid_contexto(label, pos, exception=False):
# type: (str, int, bool) -> bool
cp_value = ord(label[pos])
if cp_value == 0x00b7:
@@ -221,7 +233,8 @@ def valid_contexto(label: str, pos: int, exception: bool = False) -> bool:
return False
def check_label(label: Union[str, bytes, bytearray]) -> None:
def check_label(label):
# type: (Union[str, bytes, bytearray]) -> None
if isinstance(label, (bytes, bytearray)):
label = label.decode('utf-8')
if len(label) == 0:
@@ -252,7 +265,8 @@ def check_label(label: Union[str, bytes, bytearray]) -> None:
check_bidi(label)
def alabel(label: str) -> bytes:
def alabel(label):
# type: (str) -> bytes
try:
label_bytes = label.encode('ascii')
ulabel(label_bytes)
@@ -276,7 +290,8 @@ def alabel(label: str) -> bytes:
return label_bytes
def ulabel(label: Union[str, bytes, bytearray]) -> str:
def ulabel(label):
# type: (Union[str, bytes, bytearray]) -> str
if not isinstance(label, (bytes, bytearray)):
try:
label_bytes = label.encode('ascii')
@@ -297,15 +312,13 @@ def ulabel(label: Union[str, bytes, bytearray]) -> str:
check_label(label_bytes)
return label_bytes.decode('ascii')
try:
label = label_bytes.decode('punycode')
except UnicodeError:
raise IDNAError('Invalid A-label')
label = label_bytes.decode('punycode')
check_label(label)
return label
def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str:
def uts46_remap(domain, std3_rules=True, transitional=False):
# type: (str, bool, bool) -> str
"""Re-map the characters in the string according to UTS46 processing."""
from .uts46data import uts46data
output = ''
@@ -337,7 +350,8 @@ def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False
return unicodedata.normalize('NFC', output)
def encode(s: Union[str, bytes, bytearray], strict: bool = False, uts46: bool = False, std3_rules: bool = False, transitional: bool = False) -> bytes:
def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False):
# type: (Union[str, bytes, bytearray], bool, bool, bool, bool) -> bytes
if isinstance(s, (bytes, bytearray)):
s = s.decode('ascii')
if uts46:
@@ -367,12 +381,10 @@ def encode(s: Union[str, bytes, bytearray], strict: bool = False, uts46: bool =
return s
def decode(s: Union[str, bytes, bytearray], strict: bool = False, uts46: bool = False, std3_rules: bool = False) -> str:
try:
if isinstance(s, (bytes, bytearray)):
s = s.decode('ascii')
except UnicodeDecodeError:
raise IDNAError('Invalid ASCII in A-label')
def decode(s, strict=False, uts46=False, std3_rules=False):
# type: (Union[str, bytes, bytearray], bool, bool, bool) -> str
if isinstance(s, (bytes, bytearray)):
s = s.decode('ascii')
if uts46:
s = uts46_remap(s, std3_rules, False)
trailing_dot = False
@@ -1,6 +1,6 @@
# This file is automatically generated by tools/idna-data
__version__ = '14.0.0'
__version__ = '13.0.0'
scripts = {
'Greek': (
0x37000000374,
@@ -49,13 +49,12 @@ scripts = {
0x30210000302a,
0x30380000303c,
0x340000004dc0,
0x4e000000a000,
0x4e0000009ffd,
0xf9000000fa6e,
0xfa700000fada,
0x16fe200016fe4,
0x16ff000016ff2,
0x200000002a6e0,
0x2a7000002b739,
0x200000002a6de,
0x2a7000002b735,
0x2b7400002b81e,
0x2b8200002cea2,
0x2ceb00002ebe1,
@@ -76,7 +75,7 @@ scripts = {
'Hiragana': (
0x304100003097,
0x309d000030a0,
0x1b0010001b120,
0x1b0010001b11f,
0x1b1500001b153,
0x1f2000001f201,
),
@@ -88,11 +87,7 @@ scripts = {
0x330000003358,
0xff660000ff70,
0xff710000ff9e,
0x1aff00001aff4,
0x1aff50001affc,
0x1affd0001afff,
0x1b0000001b001,
0x1b1200001b123,
0x1b1640001b168,
),
}
@@ -410,39 +405,6 @@ joining_types = {
0x868: 68,
0x869: 82,
0x86a: 82,
0x870: 82,
0x871: 82,
0x872: 82,
0x873: 82,
0x874: 82,
0x875: 82,
0x876: 82,
0x877: 82,
0x878: 82,
0x879: 82,
0x87a: 82,
0x87b: 82,
0x87c: 82,
0x87d: 82,
0x87e: 82,
0x87f: 82,
0x880: 82,
0x881: 82,
0x882: 82,
0x883: 67,
0x884: 67,
0x885: 67,
0x886: 68,
0x887: 85,
0x888: 85,
0x889: 68,
0x88a: 68,
0x88b: 68,
0x88c: 68,
0x88d: 68,
0x88e: 82,
0x890: 85,
0x891: 85,
0x8a0: 68,
0x8a1: 68,
0x8a2: 68,
@@ -464,7 +426,6 @@ joining_types = {
0x8b2: 82,
0x8b3: 68,
0x8b4: 68,
0x8b5: 68,
0x8b6: 68,
0x8b7: 68,
0x8b8: 68,
@@ -483,7 +444,6 @@ joining_types = {
0x8c5: 68,
0x8c6: 68,
0x8c7: 68,
0x8c8: 68,
0x8e2: 85,
0x1806: 85,
0x1807: 68,
@@ -808,24 +768,6 @@ joining_types = {
0x10f52: 68,
0x10f53: 68,
0x10f54: 82,
0x10f70: 68,
0x10f71: 68,
0x10f72: 68,
0x10f73: 68,
0x10f74: 82,
0x10f75: 82,
0x10f76: 68,
0x10f77: 68,
0x10f78: 68,
0x10f79: 68,
0x10f7a: 68,
0x10f7b: 68,
0x10f7c: 68,
0x10f7d: 68,
0x10f7e: 68,
0x10f7f: 68,
0x10f80: 68,
0x10f81: 68,
0x10fb0: 68,
0x10fb1: 85,
0x10fb2: 68,
@@ -1226,9 +1168,9 @@ codepoint_classes = {
0x8000000082e,
0x8400000085c,
0x8600000086b,
0x87000000888,
0x8890000088f,
0x898000008e2,
0x8a0000008b5,
0x8b6000008c8,
0x8d3000008e2,
0x8e300000958,
0x96000000964,
0x96600000970,
@@ -1310,12 +1252,11 @@ codepoint_classes = {
0xc0e00000c11,
0xc1200000c29,
0xc2a00000c3a,
0xc3c00000c45,
0xc3d00000c45,
0xc4600000c49,
0xc4a00000c4e,
0xc5500000c57,
0xc5800000c5b,
0xc5d00000c5e,
0xc6000000c64,
0xc6600000c70,
0xc8000000c84,
@@ -1328,7 +1269,7 @@ codepoint_classes = {
0xcc600000cc9,
0xcca00000cce,
0xcd500000cd7,
0xcdd00000cdf,
0xcde00000cdf,
0xce000000ce4,
0xce600000cf0,
0xcf100000cf3,
@@ -1425,8 +1366,9 @@ codepoint_classes = {
0x16810000169b,
0x16a0000016eb,
0x16f1000016f9,
0x170000001716,
0x171f00001735,
0x17000000170d,
0x170e00001715,
0x172000001735,
0x174000001754,
0x17600000176d,
0x176e00001771,
@@ -1455,8 +1397,8 @@ codepoint_classes = {
0x1a9000001a9a,
0x1aa700001aa8,
0x1ab000001abe,
0x1abf00001acf,
0x1b0000001b4d,
0x1abf00001ac1,
0x1b0000001b4c,
0x1b5000001b5a,
0x1b6b00001b74,
0x1b8000001bf4,
@@ -1471,7 +1413,8 @@ codepoint_classes = {
0x1d4e00001d4f,
0x1d6b00001d78,
0x1d7900001d9b,
0x1dc000001e00,
0x1dc000001dfa,
0x1dfb00001e00,
0x1e0100001e02,
0x1e0300001e04,
0x1e0500001e06,
@@ -1620,7 +1563,7 @@ codepoint_classes = {
0x1ff600001ff7,
0x214e0000214f,
0x218400002185,
0x2c3000002c60,
0x2c3000002c5f,
0x2c6100002c62,
0x2c6500002c67,
0x2c6800002c69,
@@ -1709,7 +1652,8 @@ codepoint_classes = {
0x31a0000031c0,
0x31f000003200,
0x340000004dc0,
0x4e000000a48d,
0x4e0000009ffd,
0xa0000000a48d,
0xa4d00000a4fe,
0xa5000000a60d,
0xa6100000a62c,
@@ -1822,16 +1766,9 @@ codepoint_classes = {
0xa7bb0000a7bc,
0xa7bd0000a7be,
0xa7bf0000a7c0,
0xa7c10000a7c2,
0xa7c30000a7c4,
0xa7c80000a7c9,
0xa7ca0000a7cb,
0xa7d10000a7d2,
0xa7d30000a7d4,
0xa7d50000a7d6,
0xa7d70000a7d8,
0xa7d90000a7da,
0xa7f20000a7f5,
0xa7f60000a7f8,
0xa7fa0000a828,
0xa82c0000a82d,
@@ -1897,16 +1834,9 @@ codepoint_classes = {
0x104d8000104fc,
0x1050000010528,
0x1053000010564,
0x10597000105a2,
0x105a3000105b2,
0x105b3000105ba,
0x105bb000105bd,
0x1060000010737,
0x1074000010756,
0x1076000010768,
0x1078000010786,
0x10787000107b1,
0x107b2000107bb,
0x1080000010806,
0x1080800010809,
0x1080a00010836,
@@ -1946,13 +1876,11 @@ codepoint_classes = {
0x10f0000010f1d,
0x10f2700010f28,
0x10f3000010f51,
0x10f7000010f86,
0x10fb000010fc5,
0x10fe000010ff7,
0x1100000011047,
0x1106600011076,
0x1106600011070,
0x1107f000110bb,
0x110c2000110c3,
0x110d0000110e9,
0x110f0000110fa,
0x1110000011135,
@@ -2006,7 +1934,6 @@ codepoint_classes = {
0x117000001171b,
0x1171d0001172c,
0x117300001173a,
0x1174000011747,
0x118000001183b,
0x118c0000118ea,
0x118ff00011907,
@@ -2025,7 +1952,7 @@ codepoint_classes = {
0x11a4700011a48,
0x11a5000011a9a,
0x11a9d00011a9e,
0x11ab000011af9,
0x11ac000011af9,
0x11c0000011c09,
0x11c0a00011c37,
0x11c3800011c41,
@@ -2050,14 +1977,11 @@ codepoint_classes = {
0x11fb000011fb1,
0x120000001239a,
0x1248000012544,
0x12f9000012ff1,
0x130000001342f,
0x1440000014647,
0x1680000016a39,
0x16a4000016a5f,
0x16a6000016a6a,
0x16a7000016abf,
0x16ac000016aca,
0x16ad000016aee,
0x16af000016af5,
0x16b0000016b37,
@@ -2075,10 +1999,7 @@ codepoint_classes = {
0x17000000187f8,
0x1880000018cd6,
0x18d0000018d09,
0x1aff00001aff4,
0x1aff50001affc,
0x1affd0001afff,
0x1b0000001b123,
0x1b0000001b11f,
0x1b1500001b153,
0x1b1640001b168,
0x1b1700001b2fc,
@@ -2087,15 +2008,12 @@ codepoint_classes = {
0x1bc800001bc89,
0x1bc900001bc9a,
0x1bc9d0001bc9f,
0x1cf000001cf2e,
0x1cf300001cf47,
0x1da000001da37,
0x1da3b0001da6d,
0x1da750001da76,
0x1da840001da85,
0x1da9b0001daa0,
0x1daa10001dab0,
0x1df000001df1f,
0x1e0000001e007,
0x1e0080001e019,
0x1e01b0001e022,
@@ -2105,19 +2023,14 @@ codepoint_classes = {
0x1e1300001e13e,
0x1e1400001e14a,
0x1e14e0001e14f,
0x1e2900001e2af,
0x1e2c00001e2fa,
0x1e7e00001e7e7,
0x1e7e80001e7ec,
0x1e7ed0001e7ef,
0x1e7f00001e7ff,
0x1e8000001e8c5,
0x1e8d00001e8d7,
0x1e9220001e94c,
0x1e9500001e95a,
0x1fbf00001fbfa,
0x200000002a6e0,
0x2a7000002b739,
0x200000002a6de,
0x2a7000002b735,
0x2b7400002b81e,
0x2b8200002cea2,
0x2ceb00002ebe1,
@@ -8,7 +8,8 @@ in the original list?" in time O(log(# runs)).
import bisect
from typing import List, Tuple
def intranges_from_list(list_: List[int]) -> Tuple[int, ...]:
def intranges_from_list(list_):
# type: (List[int]) -> Tuple[int, ...]
"""Represent a list of integers as a sequence of ranges:
((start_0, end_0), (start_1, end_1), ...), such that the original
integers are exactly those x such that start_i <= x < end_i for some i.
@@ -29,14 +30,17 @@ def intranges_from_list(list_: List[int]) -> Tuple[int, ...]:
return tuple(ranges)
def _encode_range(start: int, end: int) -> int:
def _encode_range(start, end):
# type: (int, int) -> int
return (start << 32) | end
def _decode_range(r: int) -> Tuple[int, int]:
def _decode_range(r):
# type: (int) -> Tuple[int, int]
return (r >> 32), (r & ((1 << 32) - 1))
def intranges_contain(int_: int, ranges: Tuple[int, ...]) -> bool:
def intranges_contain(int_, ranges):
# type: (int, Tuple[int, ...]) -> bool
"""Determine if `int_` falls into one of the ranges in `ranges`."""
tuple_ = _encode_range(int_, 0)
pos = bisect.bisect_left(ranges, tuple_)
@@ -1,2 +1,2 @@
__version__ = '3.3'
__version__ = '3.2'
File diff suppressed because it is too large Load Diff
@@ -1 +1 @@
version = (1, 0, 3)
version = (1, 0, 2)
@@ -1,4 +1,5 @@
"""Fallback pure Python implementation of msgpack"""
from datetime import datetime as _DateTime
import sys
import struct
@@ -147,38 +148,6 @@ if sys.version_info < (2, 7, 6):
else:
_unpack_from = struct.unpack_from
_NO_FORMAT_USED = ""
_MSGPACK_HEADERS = {
0xC4: (1, _NO_FORMAT_USED, TYPE_BIN),
0xC5: (2, ">H", TYPE_BIN),
0xC6: (4, ">I", TYPE_BIN),
0xC7: (2, "Bb", TYPE_EXT),
0xC8: (3, ">Hb", TYPE_EXT),
0xC9: (5, ">Ib", TYPE_EXT),
0xCA: (4, ">f"),
0xCB: (8, ">d"),
0xCC: (1, _NO_FORMAT_USED),
0xCD: (2, ">H"),
0xCE: (4, ">I"),
0xCF: (8, ">Q"),
0xD0: (1, "b"),
0xD1: (2, ">h"),
0xD2: (4, ">i"),
0xD3: (8, ">q"),
0xD4: (1, "b1s", TYPE_EXT),
0xD5: (2, "b2s", TYPE_EXT),
0xD6: (4, "b4s", TYPE_EXT),
0xD7: (8, "b8s", TYPE_EXT),
0xD8: (16, "b16s", TYPE_EXT),
0xD9: (1, _NO_FORMAT_USED, TYPE_RAW),
0xDA: (2, ">H", TYPE_RAW),
0xDB: (4, ">I", TYPE_RAW),
0xDC: (2, ">H", TYPE_ARRAY),
0xDD: (4, ">I", TYPE_ARRAY),
0xDE: (2, ">H", TYPE_MAP),
0xDF: (4, ">I", TYPE_MAP),
}
class Unpacker(object):
"""Streaming unpacker.
@@ -260,7 +229,7 @@ class Unpacker(object):
Example of streaming deserialize from socket::
unpacker = Unpacker()
unpacker = Unpacker(max_buffer_size)
while True:
buf = sock.recv(1024**2)
if not buf:
@@ -385,7 +354,7 @@ class Unpacker(object):
self._buffer.extend(view)
def _consume(self):
"""Gets rid of the used parts of the buffer."""
""" Gets rid of the used parts of the buffer. """
self._stream_offset += self._buff_i - self._buf_checkpoint
self._buf_checkpoint = self._buff_i
@@ -440,7 +409,7 @@ class Unpacker(object):
self._buff_i = 0 # rollback
raise OutOfData
def _read_header(self):
def _read_header(self, execute=EX_CONSTRUCT):
typ = TYPE_IMMEDIATE
n = 0
obj = None
@@ -455,95 +424,205 @@ class Unpacker(object):
n = b & 0b00011111
typ = TYPE_RAW
if n > self._max_str_len:
raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
obj = self._read(n)
elif b & 0b11110000 == 0b10010000:
n = b & 0b00001111
typ = TYPE_ARRAY
if n > self._max_array_len:
raise ValueError(
"%s exceeds max_array_len(%s)" % (n, self._max_array_len)
)
raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
elif b & 0b11110000 == 0b10000000:
n = b & 0b00001111
typ = TYPE_MAP
if n > self._max_map_len:
raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
elif b == 0xC0:
obj = None
elif b == 0xC2:
obj = False
elif b == 0xC3:
obj = True
elif 0xC4 <= b <= 0xC6:
size, fmt, typ = _MSGPACK_HEADERS[b]
self._reserve(size)
if len(fmt) > 0:
n = _unpack_from(fmt, self._buffer, self._buff_i)[0]
else:
n = self._buffer[self._buff_i]
self._buff_i += size
elif b == 0xC4:
typ = TYPE_BIN
self._reserve(1)
n = self._buffer[self._buff_i]
self._buff_i += 1
if n > self._max_bin_len:
raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
obj = self._read(n)
elif 0xC7 <= b <= 0xC9:
size, fmt, typ = _MSGPACK_HEADERS[b]
self._reserve(size)
L, n = _unpack_from(fmt, self._buffer, self._buff_i)
self._buff_i += size
elif b == 0xC5:
typ = TYPE_BIN
self._reserve(2)
n = _unpack_from(">H", self._buffer, self._buff_i)[0]
self._buff_i += 2
if n > self._max_bin_len:
raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
obj = self._read(n)
elif b == 0xC6:
typ = TYPE_BIN
self._reserve(4)
n = _unpack_from(">I", self._buffer, self._buff_i)[0]
self._buff_i += 4
if n > self._max_bin_len:
raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
obj = self._read(n)
elif b == 0xC7: # ext 8
typ = TYPE_EXT
self._reserve(2)
L, n = _unpack_from("Bb", self._buffer, self._buff_i)
self._buff_i += 2
if L > self._max_ext_len:
raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
obj = self._read(L)
elif 0xCA <= b <= 0xD3:
size, fmt = _MSGPACK_HEADERS[b]
self._reserve(size)
if len(fmt) > 0:
obj = _unpack_from(fmt, self._buffer, self._buff_i)[0]
else:
obj = self._buffer[self._buff_i]
self._buff_i += size
elif 0xD4 <= b <= 0xD8:
size, fmt, typ = _MSGPACK_HEADERS[b]
if self._max_ext_len < size:
raise ValueError(
"%s exceeds max_ext_len(%s)" % (size, self._max_ext_len)
)
self._reserve(size + 1)
n, obj = _unpack_from(fmt, self._buffer, self._buff_i)
self._buff_i += size + 1
elif 0xD9 <= b <= 0xDB:
size, fmt, typ = _MSGPACK_HEADERS[b]
self._reserve(size)
if len(fmt) > 0:
(n,) = _unpack_from(fmt, self._buffer, self._buff_i)
else:
n = self._buffer[self._buff_i]
self._buff_i += size
elif b == 0xC8: # ext 16
typ = TYPE_EXT
self._reserve(3)
L, n = _unpack_from(">Hb", self._buffer, self._buff_i)
self._buff_i += 3
if L > self._max_ext_len:
raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
obj = self._read(L)
elif b == 0xC9: # ext 32
typ = TYPE_EXT
self._reserve(5)
L, n = _unpack_from(">Ib", self._buffer, self._buff_i)
self._buff_i += 5
if L > self._max_ext_len:
raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
obj = self._read(L)
elif b == 0xCA:
self._reserve(4)
obj = _unpack_from(">f", self._buffer, self._buff_i)[0]
self._buff_i += 4
elif b == 0xCB:
self._reserve(8)
obj = _unpack_from(">d", self._buffer, self._buff_i)[0]
self._buff_i += 8
elif b == 0xCC:
self._reserve(1)
obj = self._buffer[self._buff_i]
self._buff_i += 1
elif b == 0xCD:
self._reserve(2)
obj = _unpack_from(">H", self._buffer, self._buff_i)[0]
self._buff_i += 2
elif b == 0xCE:
self._reserve(4)
obj = _unpack_from(">I", self._buffer, self._buff_i)[0]
self._buff_i += 4
elif b == 0xCF:
self._reserve(8)
obj = _unpack_from(">Q", self._buffer, self._buff_i)[0]
self._buff_i += 8
elif b == 0xD0:
self._reserve(1)
obj = _unpack_from("b", self._buffer, self._buff_i)[0]
self._buff_i += 1
elif b == 0xD1:
self._reserve(2)
obj = _unpack_from(">h", self._buffer, self._buff_i)[0]
self._buff_i += 2
elif b == 0xD2:
self._reserve(4)
obj = _unpack_from(">i", self._buffer, self._buff_i)[0]
self._buff_i += 4
elif b == 0xD3:
self._reserve(8)
obj = _unpack_from(">q", self._buffer, self._buff_i)[0]
self._buff_i += 8
elif b == 0xD4: # fixext 1
typ = TYPE_EXT
if self._max_ext_len < 1:
raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
self._reserve(2)
n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
self._buff_i += 2
elif b == 0xD5: # fixext 2
typ = TYPE_EXT
if self._max_ext_len < 2:
raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
self._reserve(3)
n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
self._buff_i += 3
elif b == 0xD6: # fixext 4
typ = TYPE_EXT
if self._max_ext_len < 4:
raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
self._reserve(5)
n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
self._buff_i += 5
elif b == 0xD7: # fixext 8
typ = TYPE_EXT
if self._max_ext_len < 8:
raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
self._reserve(9)
n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
self._buff_i += 9
elif b == 0xD8: # fixext 16
typ = TYPE_EXT
if self._max_ext_len < 16:
raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
self._reserve(17)
n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
self._buff_i += 17
elif b == 0xD9:
typ = TYPE_RAW
self._reserve(1)
n = self._buffer[self._buff_i]
self._buff_i += 1
if n > self._max_str_len:
raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
obj = self._read(n)
elif 0xDC <= b <= 0xDD:
size, fmt, typ = _MSGPACK_HEADERS[b]
self._reserve(size)
(n,) = _unpack_from(fmt, self._buffer, self._buff_i)
self._buff_i += size
elif b == 0xDA:
typ = TYPE_RAW
self._reserve(2)
(n,) = _unpack_from(">H", self._buffer, self._buff_i)
self._buff_i += 2
if n > self._max_str_len:
raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
obj = self._read(n)
elif b == 0xDB:
typ = TYPE_RAW
self._reserve(4)
(n,) = _unpack_from(">I", self._buffer, self._buff_i)
self._buff_i += 4
if n > self._max_str_len:
raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
obj = self._read(n)
elif b == 0xDC:
typ = TYPE_ARRAY
self._reserve(2)
(n,) = _unpack_from(">H", self._buffer, self._buff_i)
self._buff_i += 2
if n > self._max_array_len:
raise ValueError(
"%s exceeds max_array_len(%s)" % (n, self._max_array_len)
)
elif 0xDE <= b <= 0xDF:
size, fmt, typ = _MSGPACK_HEADERS[b]
self._reserve(size)
(n,) = _unpack_from(fmt, self._buffer, self._buff_i)
self._buff_i += size
raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
elif b == 0xDD:
typ = TYPE_ARRAY
self._reserve(4)
(n,) = _unpack_from(">I", self._buffer, self._buff_i)
self._buff_i += 4
if n > self._max_array_len:
raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
elif b == 0xDE:
self._reserve(2)
(n,) = _unpack_from(">H", self._buffer, self._buff_i)
self._buff_i += 2
if n > self._max_map_len:
raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
typ = TYPE_MAP
elif b == 0xDF:
self._reserve(4)
(n,) = _unpack_from(">I", self._buffer, self._buff_i)
self._buff_i += 4
if n > self._max_map_len:
raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
typ = TYPE_MAP
else:
raise FormatError("Unknown header: 0x%x" % b)
return typ, n, obj
def _unpack(self, execute=EX_CONSTRUCT):
typ, n, obj = self._read_header()
typ, n, obj = self._read_header(execute)
if execute == EX_READ_ARRAY_HEADER:
if typ != TYPE_ARRAY:
@@ -874,10 +953,6 @@ class Packer(object):
obj = self._default(obj)
default_used = 1
continue
if self._datetime and check(obj, _DateTime):
raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,))
raise TypeError("Cannot serialize %r" % (obj,))
def pack(self, obj):
@@ -17,7 +17,7 @@ __title__ = "packaging"
__summary__ = "Core utilities for Python packages"
__uri__ = "https://github.com/pypa/packaging"
__version__ = "21.3"
__version__ = "21.0"
__author__ = "Donald Stufft and individual contributors"
__email__ = "donald@stufft.io"
@@ -98,7 +98,7 @@ def _get_musl_version(executable: str) -> Optional[_MuslVersion]:
with contextlib.ExitStack() as stack:
try:
f = stack.enter_context(open(executable, "rb"))
except OSError:
except IOError:
return None
ld = _parse_ld_musl_from_elf(f)
if not ld:
@@ -19,6 +19,9 @@ class InfinityType:
def __eq__(self, other: object) -> bool:
return isinstance(other, self.__class__)
def __ne__(self, other: object) -> bool:
return not isinstance(other, self.__class__)
def __gt__(self, other: object) -> bool:
return True
@@ -48,6 +51,9 @@ class NegativeInfinityType:
def __eq__(self, other: object) -> bool:
return isinstance(other, self.__class__)
def __ne__(self, other: object) -> bool:
return not isinstance(other, self.__class__)
def __gt__(self, other: object) -> bool:
return False
@@ -57,6 +57,13 @@ class BaseSpecifier(metaclass=abc.ABCMeta):
objects are equal.
"""
@abc.abstractmethod
def __ne__(self, other: object) -> bool:
"""
Returns a boolean representing whether or not the two Specifier like
objects are not equal.
"""
@abc.abstractproperty
def prereleases(self) -> Optional[bool]:
"""
@@ -112,7 +119,7 @@ class _IndividualSpecifier(BaseSpecifier):
else ""
)
return f"<{self.__class__.__name__}({str(self)!r}{pre})>"
return "<{}({!r}{})>".format(self.__class__.__name__, str(self), pre)
def __str__(self) -> str:
return "{}{}".format(*self._spec)
@@ -135,6 +142,17 @@ class _IndividualSpecifier(BaseSpecifier):
return self._canonical_spec == other._canonical_spec
def __ne__(self, other: object) -> bool:
if isinstance(other, str):
try:
other = self.__class__(str(other))
except InvalidSpecifier:
return NotImplemented
elif not isinstance(other, self.__class__):
return NotImplemented
return self._spec != other._spec
def _get_operator(self, op: str) -> CallableOperator:
operator_callable: CallableOperator = getattr(
self, f"_compare_{self._operators[op]}"
@@ -649,7 +667,7 @@ class SpecifierSet(BaseSpecifier):
else ""
)
return f"<SpecifierSet({str(self)!r}{pre})>"
return "<SpecifierSet({!r}{})>".format(str(self), pre)
def __str__(self) -> str:
return ",".join(sorted(str(s) for s in self._specs))
@@ -688,6 +706,14 @@ class SpecifierSet(BaseSpecifier):
return self._specs == other._specs
def __ne__(self, other: object) -> bool:
if isinstance(other, (str, _IndividualSpecifier)):
other = SpecifierSet(str(other))
elif not isinstance(other, SpecifierSet):
return NotImplemented
return self._specs != other._specs
def __len__(self) -> int:
return len(self._specs)
@@ -90,7 +90,7 @@ class Tag:
return f"{self._interpreter}-{self._abi}-{self._platform}"
def __repr__(self) -> str:
return f"<{self} @ {id(self)}>"
return "<{self} @ {self_id}>".format(self=self, self_id=id(self))
def parse_tag(tag: str) -> FrozenSet[Tag]:
@@ -192,7 +192,7 @@ def cpython_tags(
if not python_version:
python_version = sys.version_info[:2]
interpreter = f"cp{_version_nodot(python_version[:2])}"
interpreter = "cp{}".format(_version_nodot(python_version[:2]))
if abis is None:
if len(python_version) > 1:
@@ -207,7 +207,7 @@ def cpython_tags(
except ValueError:
pass
platforms = list(platforms or platform_tags())
platforms = list(platforms or _platform_tags())
for abi in abis:
for platform_ in platforms:
yield Tag(interpreter, abi, platform_)
@@ -251,7 +251,7 @@ def generic_tags(
interpreter = "".join([interp_name, interp_version])
if abis is None:
abis = _generic_abi()
platforms = list(platforms or platform_tags())
platforms = list(platforms or _platform_tags())
abis = list(abis)
if "none" not in abis:
abis.append("none")
@@ -268,11 +268,11 @@ def _py_interpreter_range(py_version: PythonVersion) -> Iterator[str]:
all previous versions of that major version.
"""
if len(py_version) > 1:
yield f"py{_version_nodot(py_version[:2])}"
yield f"py{py_version[0]}"
yield "py{version}".format(version=_version_nodot(py_version[:2]))
yield "py{major}".format(major=py_version[0])
if len(py_version) > 1:
for minor in range(py_version[1] - 1, -1, -1):
yield f"py{_version_nodot((py_version[0], minor))}"
yield "py{version}".format(version=_version_nodot((py_version[0], minor)))
def compatible_tags(
@@ -290,7 +290,7 @@ def compatible_tags(
"""
if not python_version:
python_version = sys.version_info[:2]
platforms = list(platforms or platform_tags())
platforms = list(platforms or _platform_tags())
for version in _py_interpreter_range(python_version):
for platform_ in platforms:
yield Tag(version, "none", platform_)
@@ -431,7 +431,7 @@ def _generic_platforms() -> Iterator[str]:
yield _normalize_string(sysconfig.get_platform())
def platform_tags() -> Iterator[str]:
def _platform_tags() -> Iterator[str]:
"""
Provides the platform tags for this installation.
"""
@@ -481,7 +481,4 @@ def sys_tags(*, warn: bool = False) -> Iterator[Tag]:
else:
yield from generic_tags()
if interp_name == "pp":
yield from compatible_tags(interpreter="pp3")
else:
yield from compatible_tags()
yield from compatible_tags()
@@ -2,22 +2,20 @@
Utilities for determining application-specific dirs. See <https://github.com/platformdirs/platformdirs> for details and
usage.
"""
from __future__ import annotations
import importlib
import os
import sys
from pathlib import Path
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Optional, Type, Union
if TYPE_CHECKING:
from pip._vendor.typing_extensions import Literal # pragma: no cover
from typing_extensions import Literal # pragma: no cover
from .api import PlatformDirsABC
from .version import __version__, __version_info__
def _set_platform_dir_class() -> type[PlatformDirsABC]:
def _set_platform_dir_class() -> Type[PlatformDirsABC]:
if os.getenv("ANDROID_DATA") == "/data" and os.getenv("ANDROID_ROOT") == "/system":
module, name = "pip._vendor.platformdirs.android", "Android"
elif sys.platform == "win32":
@@ -26,7 +24,7 @@ def _set_platform_dir_class() -> type[PlatformDirsABC]:
module, name = "pip._vendor.platformdirs.macos", "MacOS"
else:
module, name = "pip._vendor.platformdirs.unix", "Unix"
result: type[PlatformDirsABC] = getattr(importlib.import_module(module), name)
result: Type[PlatformDirsABC] = getattr(importlib.import_module(module), name)
return result
@@ -35,9 +33,9 @@ AppDirs = PlatformDirs #: Backwards compatibility with appdirs
def user_data_dir(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
roaming: bool = False,
) -> str:
"""
@@ -51,9 +49,9 @@ def user_data_dir(
def site_data_dir(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
multipath: bool = False,
) -> str:
"""
@@ -67,9 +65,9 @@ def site_data_dir(
def user_config_dir(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
roaming: bool = False,
) -> str:
"""
@@ -83,9 +81,9 @@ def user_config_dir(
def site_config_dir(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
multipath: bool = False,
) -> str:
"""
@@ -99,9 +97,9 @@ def site_config_dir(
def user_cache_dir(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
opinion: bool = True,
) -> str:
"""
@@ -115,9 +113,9 @@ def user_cache_dir(
def user_state_dir(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
roaming: bool = False,
) -> str:
"""
@@ -131,9 +129,9 @@ def user_state_dir(
def user_log_dir(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
opinion: bool = True,
) -> str:
"""
@@ -154,9 +152,9 @@ def user_documents_dir() -> str:
def user_runtime_dir(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
opinion: bool = True,
) -> str:
"""
@@ -170,9 +168,9 @@ def user_runtime_dir(
def user_data_path(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
roaming: bool = False,
) -> Path:
"""
@@ -186,9 +184,9 @@ def user_data_path(
def site_data_path(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
multipath: bool = False,
) -> Path:
"""
@@ -202,9 +200,9 @@ def site_data_path(
def user_config_path(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
roaming: bool = False,
) -> Path:
"""
@@ -218,9 +216,9 @@ def user_config_path(
def site_config_path(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
multipath: bool = False,
) -> Path:
"""
@@ -234,9 +232,9 @@ def site_config_path(
def user_cache_path(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
opinion: bool = True,
) -> Path:
"""
@@ -250,9 +248,9 @@ def user_cache_path(
def user_state_path(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
roaming: bool = False,
) -> Path:
"""
@@ -266,9 +264,9 @@ def user_state_path(
def user_log_path(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
opinion: bool = True,
) -> Path:
"""
@@ -289,9 +287,9 @@ def user_documents_path() -> Path:
def user_runtime_path(
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
opinion: bool = True,
) -> Path:
"""
@@ -1,5 +1,3 @@
from __future__ import annotations
from pip._vendor.platformdirs import PlatformDirs, __version__
PROPS = (
@@ -1,5 +1,3 @@
from __future__ import annotations
import os
import re
import sys
@@ -82,9 +80,9 @@ def _android_folder() -> str:
""":return: base folder for the Android OS"""
try:
# First try to get path to android app via pyjnius
from jnius import autoclass
from jnius import autoclass # noqa: SC200
Context = autoclass("android.content.Context") # noqa: N806
Context = autoclass("android.content.Context") # noqa: SC200
result: str = Context.getFilesDir().getParentFile().getAbsolutePath()
except Exception:
# if fails find an android folder looking path on the sys.path
@@ -103,10 +101,10 @@ def _android_documents_folder() -> str:
""":return: documents folder for the Android OS"""
# Get directories with pyjnius
try:
from jnius import autoclass
from jnius import autoclass # noqa: SC200
Context = autoclass("android.content.Context") # noqa: N806
Environment = autoclass("android.os.Environment") # noqa: N806
Context = autoclass("android.content.Context") # noqa: SC200
Environment = autoclass("android.os.Environment")
documents_dir: str = Context.getExternalFilesDir(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath()
except Exception:
documents_dir = "/storage/emulated/0/Documents"
@@ -1,9 +1,8 @@
from __future__ import annotations
import os
import sys
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Optional, Union
if sys.version_info >= (3, 8): # pragma: no branch
from typing import Literal # pragma: no cover
@@ -16,9 +15,9 @@ class PlatformDirsABC(ABC):
def __init__(
self,
appname: str | None = None,
appauthor: str | None | Literal[False] = None,
version: str | None = None,
appname: Optional[str] = None,
appauthor: Union[str, None, "Literal[False]"] = None,
version: Optional[str] = None,
roaming: bool = False,
multipath: bool = False,
opinion: bool = True,
@@ -1,5 +1,3 @@
from __future__ import annotations
import os
from .api import PlatformDirsABC
@@ -1,9 +1,8 @@
from __future__ import annotations
import os
import sys
from configparser import ConfigParser
from pathlib import Path
from typing import Optional
from .api import PlatformDirsABC
@@ -155,7 +154,7 @@ class Unix(PlatformDirsABC):
return Path(directory)
def _get_user_dirs_folder(key: str) -> str | None:
def _get_user_dirs_folder(key: str) -> Optional[str]:
"""Return directory from user-dirs.dirs config file. See https://freedesktop.org/wiki/Software/xdg-user-dirs/"""
user_dirs_config_path = os.path.join(Unix().user_config_dir, "user-dirs.dirs")
if os.path.exists(user_dirs_config_path):
@@ -1,4 +1,4 @@
""" Version information """
__version__ = "2.4.1"
__version_info__ = (2, 4, 1)
__version__ = "2.4.0"
__version_info__ = (2, 4, 0)
@@ -1,9 +1,7 @@
from __future__ import annotations
import ctypes
import os
from functools import lru_cache
from typing import Callable
from typing import Callable, Optional
from .api import PlatformDirsABC
@@ -29,7 +27,7 @@ class Windows(PlatformDirsABC):
path = os.path.normpath(get_win_folder(const))
return self._append_parts(path)
def _append_parts(self, path: str, *, opinion_value: str | None = None) -> str:
def _append_parts(self, path: str, *, opinion_value: Optional[str] = None) -> str:
params = []
if self.appname:
if self.appauthor is not False:
@@ -1,83 +0,0 @@
"""
Pygments
~~~~~~~~
Pygments is a syntax highlighting package written in Python.
It is a generic syntax highlighter for general use in all kinds of software
such as forum systems, wikis or other applications that need to prettify
source code. Highlights are:
* a wide range of common languages and markup formats is supported
* special attention is paid to details, increasing quality by a fair amount
* support for new languages and formats are added easily
* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image
formats that PIL supports, and ANSI sequences
* it is usable as a command-line tool and as a library
* ... and it highlights even Brainfuck!
The `Pygments master branch`_ is installable with ``easy_install Pygments==dev``.
.. _Pygments master branch:
https://github.com/pygments/pygments/archive/master.zip#egg=Pygments-dev
:copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from io import StringIO, BytesIO
__version__ = '2.11.2'
__docformat__ = 'restructuredtext'
__all__ = ['lex', 'format', 'highlight']
def lex(code, lexer):
"""
Lex ``code`` with ``lexer`` and return an iterable of tokens.
"""
try:
return lexer.get_tokens(code)
except TypeError as err:
if (isinstance(err.args[0], str) and
('unbound method get_tokens' in err.args[0] or
'missing 1 required positional argument' in err.args[0])):
raise TypeError('lex() argument must be a lexer instance, '
'not a class')
raise
def format(tokens, formatter, outfile=None): # pylint: disable=redefined-builtin
"""
Format a tokenlist ``tokens`` with the formatter ``formatter``.
If ``outfile`` is given and a valid file object (an object
with a ``write`` method), the result will be written to it, otherwise
it is returned as a string.
"""
try:
if not outfile:
realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
formatter.format(tokens, realoutfile)
return realoutfile.getvalue()
else:
formatter.format(tokens, outfile)
except TypeError as err:
if (isinstance(err.args[0], str) and
('unbound method format' in err.args[0] or
'missing 1 required positional argument' in err.args[0])):
raise TypeError('format() argument must be a formatter instance, '
'not a class')
raise
def highlight(code, lexer, formatter, outfile=None):
"""
Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
If ``outfile`` is given and a valid file object (an object
with a ``write`` method), the result will be written to it, otherwise
it is returned as a string.
"""
return format(lex(code, lexer), formatter, outfile)
@@ -1,17 +0,0 @@
"""
pygments.__main__
~~~~~~~~~~~~~~~~~
Main entry point for ``python -m pygments``.
:copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import sys
from pip._vendor.pygments.cmdline import main
try:
sys.exit(main(sys.argv))
except KeyboardInterrupt:
sys.exit(1)
@@ -1,663 +0,0 @@
"""
pygments.cmdline
~~~~~~~~~~~~~~~~
Command line interface.
:copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import os
import sys
import shutil
import argparse
from textwrap import dedent
from pip._vendor.pygments import __version__, highlight
from pip._vendor.pygments.util import ClassNotFound, OptionError, docstring_headline, \
guess_decode, guess_decode_from_terminal, terminal_encoding, \
UnclosingTextIOWrapper
from pip._vendor.pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename
from pip._vendor.pygments.lexers.special import TextLexer
from pip._vendor.pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
from pip._vendor.pygments.formatters import get_all_formatters, get_formatter_by_name, \
load_formatter_from_file, get_formatter_for_filename, find_formatter_class
from pip._vendor.pygments.formatters.terminal import TerminalFormatter
from pip._vendor.pygments.formatters.terminal256 import Terminal256Formatter
from pip._vendor.pygments.filters import get_all_filters, find_filter_class
from pip._vendor.pygments.styles import get_all_styles, get_style_by_name
def _parse_options(o_strs):
opts = {}
if not o_strs:
return opts
for o_str in o_strs:
if not o_str.strip():
continue
o_args = o_str.split(',')
for o_arg in o_args:
o_arg = o_arg.strip()
try:
o_key, o_val = o_arg.split('=', 1)
o_key = o_key.strip()
o_val = o_val.strip()
except ValueError:
opts[o_arg] = True
else:
opts[o_key] = o_val
return opts
def _parse_filters(f_strs):
filters = []
if not f_strs:
return filters
for f_str in f_strs:
if ':' in f_str:
fname, fopts = f_str.split(':', 1)
filters.append((fname, _parse_options([fopts])))
else:
filters.append((f_str, {}))
return filters
def _print_help(what, name):
try:
if what == 'lexer':
cls = get_lexer_by_name(name)
print("Help on the %s lexer:" % cls.name)
print(dedent(cls.__doc__))
elif what == 'formatter':
cls = find_formatter_class(name)
print("Help on the %s formatter:" % cls.name)
print(dedent(cls.__doc__))
elif what == 'filter':
cls = find_filter_class(name)
print("Help on the %s filter:" % name)
print(dedent(cls.__doc__))
return 0
except (AttributeError, ValueError):
print("%s not found!" % what, file=sys.stderr)
return 1
def _print_list(what):
if what == 'lexer':
print()
print("Lexers:")
print("~~~~~~~")
info = []
for fullname, names, exts, _ in get_all_lexers():
tup = (', '.join(names)+':', fullname,
exts and '(filenames ' + ', '.join(exts) + ')' or '')
info.append(tup)
info.sort()
for i in info:
print(('* %s\n %s %s') % i)
elif what == 'formatter':
print()
print("Formatters:")
print("~~~~~~~~~~~")
info = []
for cls in get_all_formatters():
doc = docstring_headline(cls)
tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and
'(filenames ' + ', '.join(cls.filenames) + ')' or '')
info.append(tup)
info.sort()
for i in info:
print(('* %s\n %s %s') % i)
elif what == 'filter':
print()
print("Filters:")
print("~~~~~~~~")
for name in get_all_filters():
cls = find_filter_class(name)
print("* " + name + ':')
print(" %s" % docstring_headline(cls))
elif what == 'style':
print()
print("Styles:")
print("~~~~~~~")
for name in get_all_styles():
cls = get_style_by_name(name)
print("* " + name + ':')
print(" %s" % docstring_headline(cls))
def _print_list_as_json(requested_items):
import json
result = {}
if 'lexer' in requested_items:
info = {}
for fullname, names, filenames, mimetypes in get_all_lexers():
info[fullname] = {
'aliases': names,
'filenames': filenames,
'mimetypes': mimetypes
}
result['lexers'] = info
if 'formatter' in requested_items:
info = {}
for cls in get_all_formatters():
doc = docstring_headline(cls)
info[cls.name] = {
'aliases': cls.aliases,
'filenames': cls.filenames,
'doc': doc
}
result['formatters'] = info
if 'filter' in requested_items:
info = {}
for name in get_all_filters():
cls = find_filter_class(name)
info[name] = {
'doc': docstring_headline(cls)
}
result['filters'] = info
if 'style' in requested_items:
info = {}
for name in get_all_styles():
cls = get_style_by_name(name)
info[name] = {
'doc': docstring_headline(cls)
}
result['styles'] = info
json.dump(result, sys.stdout)
def main_inner(parser, argns):
if argns.help:
parser.print_help()
return 0
if argns.V:
print('Pygments version %s, (c) 2006-2021 by Georg Brandl, Matthäus '
'Chajdas and contributors.' % __version__)
return 0
def is_only_option(opt):
return not any(v for (k, v) in vars(argns).items() if k != opt)
# handle ``pygmentize -L``
if argns.L is not None:
arg_set = set()
for k, v in vars(argns).items():
if v:
arg_set.add(k)
arg_set.discard('L')
arg_set.discard('json')
if arg_set:
parser.print_help(sys.stderr)
return 2
# print version
if not argns.json:
main(['', '-V'])
allowed_types = {'lexer', 'formatter', 'filter', 'style'}
largs = [arg.rstrip('s') for arg in argns.L]
if any(arg not in allowed_types for arg in largs):
parser.print_help(sys.stderr)
return 0
if not largs:
largs = allowed_types
if not argns.json:
for arg in largs:
_print_list(arg)
else:
_print_list_as_json(largs)
return 0
# handle ``pygmentize -H``
if argns.H:
if not is_only_option('H'):
parser.print_help(sys.stderr)
return 2
what, name = argns.H
if what not in ('lexer', 'formatter', 'filter'):
parser.print_help(sys.stderr)
return 2
return _print_help(what, name)
# parse -O options
parsed_opts = _parse_options(argns.O or [])
# parse -P options
for p_opt in argns.P or []:
try:
name, value = p_opt.split('=', 1)
except ValueError:
parsed_opts[p_opt] = True
else:
parsed_opts[name] = value
# encodings
inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
# handle ``pygmentize -N``
if argns.N:
lexer = find_lexer_class_for_filename(argns.N)
if lexer is None:
lexer = TextLexer
print(lexer.aliases[0])
return 0
# handle ``pygmentize -C``
if argns.C:
inp = sys.stdin.buffer.read()
try:
lexer = guess_lexer(inp, inencoding=inencoding)
except ClassNotFound:
lexer = TextLexer
print(lexer.aliases[0])
return 0
# handle ``pygmentize -S``
S_opt = argns.S
a_opt = argns.a
if S_opt is not None:
f_opt = argns.f
if not f_opt:
parser.print_help(sys.stderr)
return 2
if argns.l or argns.INPUTFILE:
parser.print_help(sys.stderr)
return 2
try:
parsed_opts['style'] = S_opt
fmter = get_formatter_by_name(f_opt, **parsed_opts)
except ClassNotFound as err:
print(err, file=sys.stderr)
return 1
print(fmter.get_style_defs(a_opt or ''))
return 0
# if no -S is given, -a is not allowed
if argns.a is not None:
parser.print_help(sys.stderr)
return 2
# parse -F options
F_opts = _parse_filters(argns.F or [])
# -x: allow custom (eXternal) lexers and formatters
allow_custom_lexer_formatter = bool(argns.x)
# select lexer
lexer = None
# given by name?
lexername = argns.l
if lexername:
# custom lexer, located relative to user's cwd
if allow_custom_lexer_formatter and '.py' in lexername:
try:
filename = None
name = None
if ':' in lexername:
filename, name = lexername.rsplit(':', 1)
if '.py' in name:
# This can happen on Windows: If the lexername is
# C:\lexer.py -- return to normal load path in that case
name = None
if filename and name:
lexer = load_lexer_from_file(filename, name,
**parsed_opts)
else:
lexer = load_lexer_from_file(lexername, **parsed_opts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
try:
lexer = get_lexer_by_name(lexername, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
# read input code
code = None
if argns.INPUTFILE:
if argns.s:
print('Error: -s option not usable when input file specified',
file=sys.stderr)
return 2
infn = argns.INPUTFILE
try:
with open(infn, 'rb') as infp:
code = infp.read()
except Exception as err:
print('Error: cannot read infile:', err, file=sys.stderr)
return 1
if not inencoding:
code, inencoding = guess_decode(code)
# do we have to guess the lexer?
if not lexer:
try:
lexer = get_lexer_for_filename(infn, code, **parsed_opts)
except ClassNotFound as err:
if argns.g:
try:
lexer = guess_lexer(code, **parsed_opts)
except ClassNotFound:
lexer = TextLexer(**parsed_opts)
else:
print('Error:', err, file=sys.stderr)
return 1
except OptionError as err:
print('Error:', err, file=sys.stderr)
return 1
elif not argns.s: # treat stdin as full file (-s support is later)
# read code from terminal, always in binary mode since we want to
# decode ourselves and be tolerant with it
code = sys.stdin.buffer.read() # use .buffer to get a binary stream
if not inencoding:
code, inencoding = guess_decode_from_terminal(code, sys.stdin)
# else the lexer will do the decoding
if not lexer:
try:
lexer = guess_lexer(code, **parsed_opts)
except ClassNotFound:
lexer = TextLexer(**parsed_opts)
else: # -s option needs a lexer with -l
if not lexer:
print('Error: when using -s a lexer has to be selected with -l',
file=sys.stderr)
return 2
# process filters
for fname, fopts in F_opts:
try:
lexer.add_filter(fname, **fopts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
# select formatter
outfn = argns.o
fmter = argns.f
if fmter:
# custom formatter, located relative to user's cwd
if allow_custom_lexer_formatter and '.py' in fmter:
try:
filename = None
name = None
if ':' in fmter:
# Same logic as above for custom lexer
filename, name = fmter.rsplit(':', 1)
if '.py' in name:
name = None
if filename and name:
fmter = load_formatter_from_file(filename, name,
**parsed_opts)
else:
fmter = load_formatter_from_file(fmter, **parsed_opts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
try:
fmter = get_formatter_by_name(fmter, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
if outfn:
if not fmter:
try:
fmter = get_formatter_for_filename(outfn, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
try:
outfile = open(outfn, 'wb')
except Exception as err:
print('Error: cannot open outfile:', err, file=sys.stderr)
return 1
else:
if not fmter:
if '256' in os.environ.get('TERM', ''):
fmter = Terminal256Formatter(**parsed_opts)
else:
fmter = TerminalFormatter(**parsed_opts)
outfile = sys.stdout.buffer
# determine output encoding if not explicitly selected
if not outencoding:
if outfn:
# output file? use lexer encoding for now (can still be None)
fmter.encoding = inencoding
else:
# else use terminal encoding
fmter.encoding = terminal_encoding(sys.stdout)
# provide coloring under Windows, if possible
if not outfn and sys.platform in ('win32', 'cygwin') and \
fmter.name in ('Terminal', 'Terminal256'): # pragma: no cover
# unfortunately colorama doesn't support binary streams on Py3
outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
fmter.encoding = None
try:
import pip._vendor.colorama.initialise as colorama_initialise
except ImportError:
pass
else:
outfile = colorama_initialise.wrap_stream(
outfile, convert=None, strip=None, autoreset=False, wrap=True)
# When using the LaTeX formatter and the option `escapeinside` is
# specified, we need a special lexer which collects escaped text
# before running the chosen language lexer.
escapeinside = parsed_opts.get('escapeinside', '')
if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter):
left = escapeinside[0]
right = escapeinside[1]
lexer = LatexEmbeddedLexer(left, right, lexer)
# ... and do it!
if not argns.s:
# process whole input as per normal...
try:
highlight(code, lexer, fmter, outfile)
finally:
if outfn:
outfile.close()
return 0
else:
# line by line processing of stdin (eg: for 'tail -f')...
try:
while 1:
line = sys.stdin.buffer.readline()
if not line:
break
if not inencoding:
line = guess_decode_from_terminal(line, sys.stdin)[0]
highlight(line, lexer, fmter, outfile)
if hasattr(outfile, 'flush'):
outfile.flush()
return 0
except KeyboardInterrupt: # pragma: no cover
return 0
finally:
if outfn:
outfile.close()
class HelpFormatter(argparse.HelpFormatter):
def __init__(self, prog, indent_increment=2, max_help_position=16, width=None):
if width is None:
try:
width = shutil.get_terminal_size().columns - 2
except Exception:
pass
argparse.HelpFormatter.__init__(self, prog, indent_increment,
max_help_position, width)
def main(args=sys.argv):
"""
Main command line entry point.
"""
desc = "Highlight an input file and write the result to an output file."
parser = argparse.ArgumentParser(description=desc, add_help=False,
formatter_class=HelpFormatter)
operation = parser.add_argument_group('Main operation')
lexersel = operation.add_mutually_exclusive_group()
lexersel.add_argument(
'-l', metavar='LEXER',
help='Specify the lexer to use. (Query names with -L.) If not '
'given and -g is not present, the lexer is guessed from the filename.')
lexersel.add_argument(
'-g', action='store_true',
help='Guess the lexer from the file contents, or pass through '
'as plain text if nothing can be guessed.')
operation.add_argument(
'-F', metavar='FILTER[:options]', action='append',
help='Add a filter to the token stream. (Query names with -L.) '
'Filter options are given after a colon if necessary.')
operation.add_argument(
'-f', metavar='FORMATTER',
help='Specify the formatter to use. (Query names with -L.) '
'If not given, the formatter is guessed from the output filename, '
'and defaults to the terminal formatter if the output is to the '
'terminal or an unknown file extension.')
operation.add_argument(
'-O', metavar='OPTION=value[,OPTION=value,...]', action='append',
help='Give options to the lexer and formatter as a comma-separated '
'list of key-value pairs. '
'Example: `-O bg=light,python=cool`.')
operation.add_argument(
'-P', metavar='OPTION=value', action='append',
help='Give a single option to the lexer and formatter - with this '
'you can pass options whose value contains commas and equal signs. '
'Example: `-P "heading=Pygments, the Python highlighter"`.')
operation.add_argument(
'-o', metavar='OUTPUTFILE',
help='Where to write the output. Defaults to standard output.')
operation.add_argument(
'INPUTFILE', nargs='?',
help='Where to read the input. Defaults to standard input.')
flags = parser.add_argument_group('Operation flags')
flags.add_argument(
'-v', action='store_true',
help='Print a detailed traceback on unhandled exceptions, which '
'is useful for debugging and bug reports.')
flags.add_argument(
'-s', action='store_true',
help='Process lines one at a time until EOF, rather than waiting to '
'process the entire file. This only works for stdin, only for lexers '
'with no line-spanning constructs, and is intended for streaming '
'input such as you get from `tail -f`. '
'Example usage: `tail -f sql.log | pygmentize -s -l sql`.')
flags.add_argument(
'-x', action='store_true',
help='Allow custom lexers and formatters to be loaded from a .py file '
'relative to the current working directory. For example, '
'`-l ./customlexer.py -x`. By default, this option expects a file '
'with a class named CustomLexer or CustomFormatter; you can also '
'specify your own class name with a colon (`-l ./lexer.py:MyLexer`). '
'Users should be very careful not to use this option with untrusted '
'files, because it will import and run them.')
flags.add_argument('--json', help='Output as JSON. This can '
'be only used in conjunction with -L.',
default=False,
action='store_true')
special_modes_group = parser.add_argument_group(
'Special modes - do not do any highlighting')
special_modes = special_modes_group.add_mutually_exclusive_group()
special_modes.add_argument(
'-S', metavar='STYLE -f formatter',
help='Print style definitions for STYLE for a formatter '
'given with -f. The argument given by -a is formatter '
'dependent.')
special_modes.add_argument(
'-L', nargs='*', metavar='WHAT',
help='List lexers, formatters, styles or filters -- '
'give additional arguments for the thing(s) you want to list '
'(e.g. "styles"), or omit them to list everything.')
special_modes.add_argument(
'-N', metavar='FILENAME',
help='Guess and print out a lexer name based solely on the given '
'filename. Does not take input or highlight anything. If no specific '
'lexer can be determined, "text" is printed.')
special_modes.add_argument(
'-C', action='store_true',
help='Like -N, but print out a lexer name based solely on '
'a given content from standard input.')
special_modes.add_argument(
'-H', action='store', nargs=2, metavar=('NAME', 'TYPE'),
help='Print detailed help for the object <name> of type <type>, '
'where <type> is one of "lexer", "formatter" or "filter".')
special_modes.add_argument(
'-V', action='store_true',
help='Print the package version.')
special_modes.add_argument(
'-h', '--help', action='store_true',
help='Print this help.')
special_modes_group.add_argument(
'-a', metavar='ARG',
help='Formatter-specific additional argument for the -S (print '
'style sheet) mode.')
argns = parser.parse_args(args[1:])
try:
return main_inner(parser, argns)
except Exception:
if argns.v:
print(file=sys.stderr)
print('*' * 65, file=sys.stderr)
print('An unhandled exception occurred while highlighting.',
file=sys.stderr)
print('Please report the whole traceback to the issue tracker at',
file=sys.stderr)
print('<https://github.com/pygments/pygments/issues>.',
file=sys.stderr)
print('*' * 65, file=sys.stderr)
print(file=sys.stderr)
raise
import traceback
info = traceback.format_exception(*sys.exc_info())
msg = info[-1].strip()
if len(info) >= 3:
# extract relevant file and position info
msg += '\n (f%s)' % info[-2].split('\n')[0].strip()[1:]
print(file=sys.stderr)
print('*** Error while highlighting:', file=sys.stderr)
print(msg, file=sys.stderr)
print('*** If this is a bug you want to report, please rerun with -v.',
file=sys.stderr)
return 1
@@ -1,70 +0,0 @@
"""
pygments.console
~~~~~~~~~~~~~~~~
Format colored console output.
:copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
esc = "\x1b["
codes = {}
codes[""] = ""
codes["reset"] = esc + "39;49;00m"
codes["bold"] = esc + "01m"
codes["faint"] = esc + "02m"
codes["standout"] = esc + "03m"
codes["underline"] = esc + "04m"
codes["blink"] = esc + "05m"
codes["overline"] = esc + "06m"
dark_colors = ["black", "red", "green", "yellow", "blue",
"magenta", "cyan", "gray"]
light_colors = ["brightblack", "brightred", "brightgreen", "brightyellow", "brightblue",
"brightmagenta", "brightcyan", "white"]
x = 30
for d, l in zip(dark_colors, light_colors):
codes[d] = esc + "%im" % x
codes[l] = esc + "%im" % (60 + x)
x += 1
del d, l, x
codes["white"] = codes["bold"]
def reset_color():
return codes["reset"]
def colorize(color_key, text):
return codes[color_key] + text + codes["reset"]
def ansiformat(attr, text):
"""
Format ``text`` with a color and/or some attributes::
color normal color
*color* bold color
_color_ underlined color
+color+ blinking color
"""
result = []
if attr[:1] == attr[-1:] == '+':
result.append(codes['blink'])
attr = attr[1:-1]
if attr[:1] == attr[-1:] == '*':
result.append(codes['bold'])
attr = attr[1:-1]
if attr[:1] == attr[-1:] == '_':
result.append(codes['underline'])
attr = attr[1:-1]
result.append(codes[attr])
result.append(text)
result.append(codes['reset'])
return ''.join(result)
@@ -1,71 +0,0 @@
"""
pygments.filter
~~~~~~~~~~~~~~~
Module that implements the default filter.
:copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
def apply_filters(stream, filters, lexer=None):
"""
Use this method to apply an iterable of filters to
a stream. If lexer is given it's forwarded to the
filter, otherwise the filter receives `None`.
"""
def _apply(filter_, stream):
yield from filter_.filter(lexer, stream)
for filter_ in filters:
stream = _apply(filter_, stream)
return stream
def simplefilter(f):
"""
Decorator that converts a function into a filter::
@simplefilter
def lowercase(self, lexer, stream, options):
for ttype, value in stream:
yield ttype, value.lower()
"""
return type(f.__name__, (FunctionFilter,), {
'__module__': getattr(f, '__module__'),
'__doc__': f.__doc__,
'function': f,
})
class Filter:
"""
Default filter. Subclass this class or use the `simplefilter`
decorator to create own filters.
"""
def __init__(self, **options):
self.options = options
def filter(self, lexer, stream):
raise NotImplementedError()
class FunctionFilter(Filter):
"""
Abstract class used by `simplefilter` to create simple
function filters on the fly. The `simplefilter` decorator
automatically creates subclasses of this class for
functions passed to it.
"""
function = None
def __init__(self, **options):
if not hasattr(self, 'function'):
raise TypeError('%r used without bound function' %
self.__class__.__name__)
Filter.__init__(self, **options)
def filter(self, lexer, stream):
# pylint: disable=not-callable
yield from self.function(lexer, stream, self.options)
@@ -1,937 +0,0 @@
"""
pygments.filters
~~~~~~~~~~~~~~~~
Module containing filter lookup functions and default
filters.
:copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pip._vendor.pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \
string_to_tokentype
from pip._vendor.pygments.filter import Filter
from pip._vendor.pygments.util import get_list_opt, get_int_opt, get_bool_opt, \
get_choice_opt, ClassNotFound, OptionError
from pip._vendor.pygments.plugin import find_plugin_filters
def find_filter_class(filtername):
"""Lookup a filter by name. Return None if not found."""
if filtername in FILTERS:
return FILTERS[filtername]
for name, cls in find_plugin_filters():
if name == filtername:
return cls
return None
def get_filter_by_name(filtername, **options):
"""Return an instantiated filter.
Options are passed to the filter initializer if wanted.
Raise a ClassNotFound if not found.
"""
cls = find_filter_class(filtername)
if cls:
return cls(**options)
else:
raise ClassNotFound('filter %r not found' % filtername)
def get_all_filters():
"""Return a generator of all filter names."""
yield from FILTERS
for name, _ in find_plugin_filters():
yield name
def _replace_special(ttype, value, regex, specialttype,
replacefunc=lambda x: x):
last = 0
for match in regex.finditer(value):
start, end = match.start(), match.end()
if start != last:
yield ttype, value[last:start]
yield specialttype, replacefunc(value[start:end])
last = end
if last != len(value):
yield ttype, value[last:]
class CodeTagFilter(Filter):
"""Highlight special code tags in comments and docstrings.
Options accepted:
`codetags` : list of strings
A list of strings that are flagged as code tags. The default is to
highlight ``XXX``, ``TODO``, ``BUG`` and ``NOTE``.
"""
def __init__(self, **options):
Filter.__init__(self, **options)
tags = get_list_opt(options, 'codetags',
['XXX', 'TODO', 'BUG', 'NOTE'])
self.tag_re = re.compile(r'\b(%s)\b' % '|'.join([
re.escape(tag) for tag in tags if tag
]))
def filter(self, lexer, stream):
regex = self.tag_re
for ttype, value in stream:
if ttype in String.Doc or \
ttype in Comment and \
ttype not in Comment.Preproc:
yield from _replace_special(ttype, value, regex, Comment.Special)
else:
yield ttype, value
class SymbolFilter(Filter):
"""Convert mathematical symbols such as \\<longrightarrow> in Isabelle
or \\longrightarrow in LaTeX into Unicode characters.
This is mostly useful for HTML or console output when you want to
approximate the source rendering you'd see in an IDE.
Options accepted:
`lang` : string
The symbol language. Must be one of ``'isabelle'`` or
``'latex'``. The default is ``'isabelle'``.
"""
latex_symbols = {
'\\alpha' : '\U000003b1',
'\\beta' : '\U000003b2',
'\\gamma' : '\U000003b3',
'\\delta' : '\U000003b4',
'\\varepsilon' : '\U000003b5',
'\\zeta' : '\U000003b6',
'\\eta' : '\U000003b7',
'\\vartheta' : '\U000003b8',
'\\iota' : '\U000003b9',
'\\kappa' : '\U000003ba',
'\\lambda' : '\U000003bb',
'\\mu' : '\U000003bc',
'\\nu' : '\U000003bd',
'\\xi' : '\U000003be',
'\\pi' : '\U000003c0',
'\\varrho' : '\U000003c1',
'\\sigma' : '\U000003c3',
'\\tau' : '\U000003c4',
'\\upsilon' : '\U000003c5',
'\\varphi' : '\U000003c6',
'\\chi' : '\U000003c7',
'\\psi' : '\U000003c8',
'\\omega' : '\U000003c9',
'\\Gamma' : '\U00000393',
'\\Delta' : '\U00000394',
'\\Theta' : '\U00000398',
'\\Lambda' : '\U0000039b',
'\\Xi' : '\U0000039e',
'\\Pi' : '\U000003a0',
'\\Sigma' : '\U000003a3',
'\\Upsilon' : '\U000003a5',
'\\Phi' : '\U000003a6',
'\\Psi' : '\U000003a8',
'\\Omega' : '\U000003a9',
'\\leftarrow' : '\U00002190',
'\\longleftarrow' : '\U000027f5',
'\\rightarrow' : '\U00002192',
'\\longrightarrow' : '\U000027f6',
'\\Leftarrow' : '\U000021d0',
'\\Longleftarrow' : '\U000027f8',
'\\Rightarrow' : '\U000021d2',
'\\Longrightarrow' : '\U000027f9',
'\\leftrightarrow' : '\U00002194',
'\\longleftrightarrow' : '\U000027f7',
'\\Leftrightarrow' : '\U000021d4',
'\\Longleftrightarrow' : '\U000027fa',
'\\mapsto' : '\U000021a6',
'\\longmapsto' : '\U000027fc',
'\\relbar' : '\U00002500',
'\\Relbar' : '\U00002550',
'\\hookleftarrow' : '\U000021a9',
'\\hookrightarrow' : '\U000021aa',
'\\leftharpoondown' : '\U000021bd',
'\\rightharpoondown' : '\U000021c1',
'\\leftharpoonup' : '\U000021bc',
'\\rightharpoonup' : '\U000021c0',
'\\rightleftharpoons' : '\U000021cc',
'\\leadsto' : '\U0000219d',
'\\downharpoonleft' : '\U000021c3',
'\\downharpoonright' : '\U000021c2',
'\\upharpoonleft' : '\U000021bf',
'\\upharpoonright' : '\U000021be',
'\\restriction' : '\U000021be',
'\\uparrow' : '\U00002191',
'\\Uparrow' : '\U000021d1',
'\\downarrow' : '\U00002193',
'\\Downarrow' : '\U000021d3',
'\\updownarrow' : '\U00002195',
'\\Updownarrow' : '\U000021d5',
'\\langle' : '\U000027e8',
'\\rangle' : '\U000027e9',
'\\lceil' : '\U00002308',
'\\rceil' : '\U00002309',
'\\lfloor' : '\U0000230a',
'\\rfloor' : '\U0000230b',
'\\flqq' : '\U000000ab',
'\\frqq' : '\U000000bb',
'\\bot' : '\U000022a5',
'\\top' : '\U000022a4',
'\\wedge' : '\U00002227',
'\\bigwedge' : '\U000022c0',
'\\vee' : '\U00002228',
'\\bigvee' : '\U000022c1',
'\\forall' : '\U00002200',
'\\exists' : '\U00002203',
'\\nexists' : '\U00002204',
'\\neg' : '\U000000ac',
'\\Box' : '\U000025a1',
'\\Diamond' : '\U000025c7',
'\\vdash' : '\U000022a2',
'\\models' : '\U000022a8',
'\\dashv' : '\U000022a3',
'\\surd' : '\U0000221a',
'\\le' : '\U00002264',
'\\ge' : '\U00002265',
'\\ll' : '\U0000226a',
'\\gg' : '\U0000226b',
'\\lesssim' : '\U00002272',
'\\gtrsim' : '\U00002273',
'\\lessapprox' : '\U00002a85',
'\\gtrapprox' : '\U00002a86',
'\\in' : '\U00002208',
'\\notin' : '\U00002209',
'\\subset' : '\U00002282',
'\\supset' : '\U00002283',
'\\subseteq' : '\U00002286',
'\\supseteq' : '\U00002287',
'\\sqsubset' : '\U0000228f',
'\\sqsupset' : '\U00002290',
'\\sqsubseteq' : '\U00002291',
'\\sqsupseteq' : '\U00002292',
'\\cap' : '\U00002229',
'\\bigcap' : '\U000022c2',
'\\cup' : '\U0000222a',
'\\bigcup' : '\U000022c3',
'\\sqcup' : '\U00002294',
'\\bigsqcup' : '\U00002a06',
'\\sqcap' : '\U00002293',
'\\Bigsqcap' : '\U00002a05',
'\\setminus' : '\U00002216',
'\\propto' : '\U0000221d',
'\\uplus' : '\U0000228e',
'\\bigplus' : '\U00002a04',
'\\sim' : '\U0000223c',
'\\doteq' : '\U00002250',
'\\simeq' : '\U00002243',
'\\approx' : '\U00002248',
'\\asymp' : '\U0000224d',
'\\cong' : '\U00002245',
'\\equiv' : '\U00002261',
'\\Join' : '\U000022c8',
'\\bowtie' : '\U00002a1d',
'\\prec' : '\U0000227a',
'\\succ' : '\U0000227b',
'\\preceq' : '\U0000227c',
'\\succeq' : '\U0000227d',
'\\parallel' : '\U00002225',
'\\mid' : '\U000000a6',
'\\pm' : '\U000000b1',
'\\mp' : '\U00002213',
'\\times' : '\U000000d7',
'\\div' : '\U000000f7',
'\\cdot' : '\U000022c5',
'\\star' : '\U000022c6',
'\\circ' : '\U00002218',
'\\dagger' : '\U00002020',
'\\ddagger' : '\U00002021',
'\\lhd' : '\U000022b2',
'\\rhd' : '\U000022b3',
'\\unlhd' : '\U000022b4',
'\\unrhd' : '\U000022b5',
'\\triangleleft' : '\U000025c3',
'\\triangleright' : '\U000025b9',
'\\triangle' : '\U000025b3',
'\\triangleq' : '\U0000225c',
'\\oplus' : '\U00002295',
'\\bigoplus' : '\U00002a01',
'\\otimes' : '\U00002297',
'\\bigotimes' : '\U00002a02',
'\\odot' : '\U00002299',
'\\bigodot' : '\U00002a00',
'\\ominus' : '\U00002296',
'\\oslash' : '\U00002298',
'\\dots' : '\U00002026',
'\\cdots' : '\U000022ef',
'\\sum' : '\U00002211',
'\\prod' : '\U0000220f',
'\\coprod' : '\U00002210',
'\\infty' : '\U0000221e',
'\\int' : '\U0000222b',
'\\oint' : '\U0000222e',
'\\clubsuit' : '\U00002663',
'\\diamondsuit' : '\U00002662',
'\\heartsuit' : '\U00002661',
'\\spadesuit' : '\U00002660',
'\\aleph' : '\U00002135',
'\\emptyset' : '\U00002205',
'\\nabla' : '\U00002207',
'\\partial' : '\U00002202',
'\\flat' : '\U0000266d',
'\\natural' : '\U0000266e',
'\\sharp' : '\U0000266f',
'\\angle' : '\U00002220',
'\\copyright' : '\U000000a9',
'\\textregistered' : '\U000000ae',
'\\textonequarter' : '\U000000bc',
'\\textonehalf' : '\U000000bd',
'\\textthreequarters' : '\U000000be',
'\\textordfeminine' : '\U000000aa',
'\\textordmasculine' : '\U000000ba',
'\\euro' : '\U000020ac',
'\\pounds' : '\U000000a3',
'\\yen' : '\U000000a5',
'\\textcent' : '\U000000a2',
'\\textcurrency' : '\U000000a4',
'\\textdegree' : '\U000000b0',
}
isabelle_symbols = {
'\\<zero>' : '\U0001d7ec',
'\\<one>' : '\U0001d7ed',
'\\<two>' : '\U0001d7ee',
'\\<three>' : '\U0001d7ef',
'\\<four>' : '\U0001d7f0',
'\\<five>' : '\U0001d7f1',
'\\<six>' : '\U0001d7f2',
'\\<seven>' : '\U0001d7f3',
'\\<eight>' : '\U0001d7f4',
'\\<nine>' : '\U0001d7f5',
'\\<A>' : '\U0001d49c',
'\\<B>' : '\U0000212c',
'\\<C>' : '\U0001d49e',
'\\<D>' : '\U0001d49f',
'\\<E>' : '\U00002130',
'\\<F>' : '\U00002131',
'\\<G>' : '\U0001d4a2',
'\\<H>' : '\U0000210b',
'\\<I>' : '\U00002110',
'\\<J>' : '\U0001d4a5',
'\\<K>' : '\U0001d4a6',
'\\<L>' : '\U00002112',
'\\<M>' : '\U00002133',
'\\<N>' : '\U0001d4a9',
'\\<O>' : '\U0001d4aa',
'\\<P>' : '\U0001d4ab',
'\\<Q>' : '\U0001d4ac',
'\\<R>' : '\U0000211b',
'\\<S>' : '\U0001d4ae',
'\\<T>' : '\U0001d4af',
'\\<U>' : '\U0001d4b0',
'\\<V>' : '\U0001d4b1',
'\\<W>' : '\U0001d4b2',
'\\<X>' : '\U0001d4b3',
'\\<Y>' : '\U0001d4b4',
'\\<Z>' : '\U0001d4b5',
'\\<a>' : '\U0001d5ba',
'\\<b>' : '\U0001d5bb',
'\\<c>' : '\U0001d5bc',
'\\<d>' : '\U0001d5bd',
'\\<e>' : '\U0001d5be',
'\\<f>' : '\U0001d5bf',
'\\<g>' : '\U0001d5c0',
'\\<h>' : '\U0001d5c1',
'\\<i>' : '\U0001d5c2',
'\\<j>' : '\U0001d5c3',
'\\<k>' : '\U0001d5c4',
'\\<l>' : '\U0001d5c5',
'\\<m>' : '\U0001d5c6',
'\\<n>' : '\U0001d5c7',
'\\<o>' : '\U0001d5c8',
'\\<p>' : '\U0001d5c9',
'\\<q>' : '\U0001d5ca',
'\\<r>' : '\U0001d5cb',
'\\<s>' : '\U0001d5cc',
'\\<t>' : '\U0001d5cd',
'\\<u>' : '\U0001d5ce',
'\\<v>' : '\U0001d5cf',
'\\<w>' : '\U0001d5d0',
'\\<x>' : '\U0001d5d1',
'\\<y>' : '\U0001d5d2',
'\\<z>' : '\U0001d5d3',
'\\<AA>' : '\U0001d504',
'\\<BB>' : '\U0001d505',
'\\<CC>' : '\U0000212d',
'\\<DD>' : '\U0001d507',
'\\<EE>' : '\U0001d508',
'\\<FF>' : '\U0001d509',
'\\<GG>' : '\U0001d50a',
'\\<HH>' : '\U0000210c',
'\\<II>' : '\U00002111',
'\\<JJ>' : '\U0001d50d',
'\\<KK>' : '\U0001d50e',
'\\<LL>' : '\U0001d50f',
'\\<MM>' : '\U0001d510',
'\\<NN>' : '\U0001d511',
'\\<OO>' : '\U0001d512',
'\\<PP>' : '\U0001d513',
'\\<QQ>' : '\U0001d514',
'\\<RR>' : '\U0000211c',
'\\<SS>' : '\U0001d516',
'\\<TT>' : '\U0001d517',
'\\<UU>' : '\U0001d518',
'\\<VV>' : '\U0001d519',
'\\<WW>' : '\U0001d51a',
'\\<XX>' : '\U0001d51b',
'\\<YY>' : '\U0001d51c',
'\\<ZZ>' : '\U00002128',
'\\<aa>' : '\U0001d51e',
'\\<bb>' : '\U0001d51f',
'\\<cc>' : '\U0001d520',
'\\<dd>' : '\U0001d521',
'\\<ee>' : '\U0001d522',
'\\<ff>' : '\U0001d523',
'\\<gg>' : '\U0001d524',
'\\<hh>' : '\U0001d525',
'\\<ii>' : '\U0001d526',
'\\<jj>' : '\U0001d527',
'\\<kk>' : '\U0001d528',
'\\<ll>' : '\U0001d529',
'\\<mm>' : '\U0001d52a',
'\\<nn>' : '\U0001d52b',
'\\<oo>' : '\U0001d52c',
'\\<pp>' : '\U0001d52d',
'\\<qq>' : '\U0001d52e',
'\\<rr>' : '\U0001d52f',
'\\<ss>' : '\U0001d530',
'\\<tt>' : '\U0001d531',
'\\<uu>' : '\U0001d532',
'\\<vv>' : '\U0001d533',
'\\<ww>' : '\U0001d534',
'\\<xx>' : '\U0001d535',
'\\<yy>' : '\U0001d536',
'\\<zz>' : '\U0001d537',
'\\<alpha>' : '\U000003b1',
'\\<beta>' : '\U000003b2',
'\\<gamma>' : '\U000003b3',
'\\<delta>' : '\U000003b4',
'\\<epsilon>' : '\U000003b5',
'\\<zeta>' : '\U000003b6',
'\\<eta>' : '\U000003b7',
'\\<theta>' : '\U000003b8',
'\\<iota>' : '\U000003b9',
'\\<kappa>' : '\U000003ba',
'\\<lambda>' : '\U000003bb',
'\\<mu>' : '\U000003bc',
'\\<nu>' : '\U000003bd',
'\\<xi>' : '\U000003be',
'\\<pi>' : '\U000003c0',
'\\<rho>' : '\U000003c1',
'\\<sigma>' : '\U000003c3',
'\\<tau>' : '\U000003c4',
'\\<upsilon>' : '\U000003c5',
'\\<phi>' : '\U000003c6',
'\\<chi>' : '\U000003c7',
'\\<psi>' : '\U000003c8',
'\\<omega>' : '\U000003c9',
'\\<Gamma>' : '\U00000393',
'\\<Delta>' : '\U00000394',
'\\<Theta>' : '\U00000398',
'\\<Lambda>' : '\U0000039b',
'\\<Xi>' : '\U0000039e',
'\\<Pi>' : '\U000003a0',
'\\<Sigma>' : '\U000003a3',
'\\<Upsilon>' : '\U000003a5',
'\\<Phi>' : '\U000003a6',
'\\<Psi>' : '\U000003a8',
'\\<Omega>' : '\U000003a9',
'\\<bool>' : '\U0001d539',
'\\<complex>' : '\U00002102',
'\\<nat>' : '\U00002115',
'\\<rat>' : '\U0000211a',
'\\<real>' : '\U0000211d',
'\\<int>' : '\U00002124',
'\\<leftarrow>' : '\U00002190',
'\\<longleftarrow>' : '\U000027f5',
'\\<rightarrow>' : '\U00002192',
'\\<longrightarrow>' : '\U000027f6',
'\\<Leftarrow>' : '\U000021d0',
'\\<Longleftarrow>' : '\U000027f8',
'\\<Rightarrow>' : '\U000021d2',
'\\<Longrightarrow>' : '\U000027f9',
'\\<leftrightarrow>' : '\U00002194',
'\\<longleftrightarrow>' : '\U000027f7',
'\\<Leftrightarrow>' : '\U000021d4',
'\\<Longleftrightarrow>' : '\U000027fa',
'\\<mapsto>' : '\U000021a6',
'\\<longmapsto>' : '\U000027fc',
'\\<midarrow>' : '\U00002500',
'\\<Midarrow>' : '\U00002550',
'\\<hookleftarrow>' : '\U000021a9',
'\\<hookrightarrow>' : '\U000021aa',
'\\<leftharpoondown>' : '\U000021bd',
'\\<rightharpoondown>' : '\U000021c1',
'\\<leftharpoonup>' : '\U000021bc',
'\\<rightharpoonup>' : '\U000021c0',
'\\<rightleftharpoons>' : '\U000021cc',
'\\<leadsto>' : '\U0000219d',
'\\<downharpoonleft>' : '\U000021c3',
'\\<downharpoonright>' : '\U000021c2',
'\\<upharpoonleft>' : '\U000021bf',
'\\<upharpoonright>' : '\U000021be',
'\\<restriction>' : '\U000021be',
'\\<Colon>' : '\U00002237',
'\\<up>' : '\U00002191',
'\\<Up>' : '\U000021d1',
'\\<down>' : '\U00002193',
'\\<Down>' : '\U000021d3',
'\\<updown>' : '\U00002195',
'\\<Updown>' : '\U000021d5',
'\\<langle>' : '\U000027e8',
'\\<rangle>' : '\U000027e9',
'\\<lceil>' : '\U00002308',
'\\<rceil>' : '\U00002309',
'\\<lfloor>' : '\U0000230a',
'\\<rfloor>' : '\U0000230b',
'\\<lparr>' : '\U00002987',
'\\<rparr>' : '\U00002988',
'\\<lbrakk>' : '\U000027e6',
'\\<rbrakk>' : '\U000027e7',
'\\<lbrace>' : '\U00002983',
'\\<rbrace>' : '\U00002984',
'\\<guillemotleft>' : '\U000000ab',
'\\<guillemotright>' : '\U000000bb',
'\\<bottom>' : '\U000022a5',
'\\<top>' : '\U000022a4',
'\\<and>' : '\U00002227',
'\\<And>' : '\U000022c0',
'\\<or>' : '\U00002228',
'\\<Or>' : '\U000022c1',
'\\<forall>' : '\U00002200',
'\\<exists>' : '\U00002203',
'\\<nexists>' : '\U00002204',
'\\<not>' : '\U000000ac',
'\\<box>' : '\U000025a1',
'\\<diamond>' : '\U000025c7',
'\\<turnstile>' : '\U000022a2',
'\\<Turnstile>' : '\U000022a8',
'\\<tturnstile>' : '\U000022a9',
'\\<TTurnstile>' : '\U000022ab',
'\\<stileturn>' : '\U000022a3',
'\\<surd>' : '\U0000221a',
'\\<le>' : '\U00002264',
'\\<ge>' : '\U00002265',
'\\<lless>' : '\U0000226a',
'\\<ggreater>' : '\U0000226b',
'\\<lesssim>' : '\U00002272',
'\\<greatersim>' : '\U00002273',
'\\<lessapprox>' : '\U00002a85',
'\\<greaterapprox>' : '\U00002a86',
'\\<in>' : '\U00002208',
'\\<notin>' : '\U00002209',
'\\<subset>' : '\U00002282',
'\\<supset>' : '\U00002283',
'\\<subseteq>' : '\U00002286',
'\\<supseteq>' : '\U00002287',
'\\<sqsubset>' : '\U0000228f',
'\\<sqsupset>' : '\U00002290',
'\\<sqsubseteq>' : '\U00002291',
'\\<sqsupseteq>' : '\U00002292',
'\\<inter>' : '\U00002229',
'\\<Inter>' : '\U000022c2',
'\\<union>' : '\U0000222a',
'\\<Union>' : '\U000022c3',
'\\<squnion>' : '\U00002294',
'\\<Squnion>' : '\U00002a06',
'\\<sqinter>' : '\U00002293',
'\\<Sqinter>' : '\U00002a05',
'\\<setminus>' : '\U00002216',
'\\<propto>' : '\U0000221d',
'\\<uplus>' : '\U0000228e',
'\\<Uplus>' : '\U00002a04',
'\\<noteq>' : '\U00002260',
'\\<sim>' : '\U0000223c',
'\\<doteq>' : '\U00002250',
'\\<simeq>' : '\U00002243',
'\\<approx>' : '\U00002248',
'\\<asymp>' : '\U0000224d',
'\\<cong>' : '\U00002245',
'\\<smile>' : '\U00002323',
'\\<equiv>' : '\U00002261',
'\\<frown>' : '\U00002322',
'\\<Join>' : '\U000022c8',
'\\<bowtie>' : '\U00002a1d',
'\\<prec>' : '\U0000227a',
'\\<succ>' : '\U0000227b',
'\\<preceq>' : '\U0000227c',
'\\<succeq>' : '\U0000227d',
'\\<parallel>' : '\U00002225',
'\\<bar>' : '\U000000a6',
'\\<plusminus>' : '\U000000b1',
'\\<minusplus>' : '\U00002213',
'\\<times>' : '\U000000d7',
'\\<div>' : '\U000000f7',
'\\<cdot>' : '\U000022c5',
'\\<star>' : '\U000022c6',
'\\<bullet>' : '\U00002219',
'\\<circ>' : '\U00002218',
'\\<dagger>' : '\U00002020',
'\\<ddagger>' : '\U00002021',
'\\<lhd>' : '\U000022b2',
'\\<rhd>' : '\U000022b3',
'\\<unlhd>' : '\U000022b4',
'\\<unrhd>' : '\U000022b5',
'\\<triangleleft>' : '\U000025c3',
'\\<triangleright>' : '\U000025b9',
'\\<triangle>' : '\U000025b3',
'\\<triangleq>' : '\U0000225c',
'\\<oplus>' : '\U00002295',
'\\<Oplus>' : '\U00002a01',
'\\<otimes>' : '\U00002297',
'\\<Otimes>' : '\U00002a02',
'\\<odot>' : '\U00002299',
'\\<Odot>' : '\U00002a00',
'\\<ominus>' : '\U00002296',
'\\<oslash>' : '\U00002298',
'\\<dots>' : '\U00002026',
'\\<cdots>' : '\U000022ef',
'\\<Sum>' : '\U00002211',
'\\<Prod>' : '\U0000220f',
'\\<Coprod>' : '\U00002210',
'\\<infinity>' : '\U0000221e',
'\\<integral>' : '\U0000222b',
'\\<ointegral>' : '\U0000222e',
'\\<clubsuit>' : '\U00002663',
'\\<diamondsuit>' : '\U00002662',
'\\<heartsuit>' : '\U00002661',
'\\<spadesuit>' : '\U00002660',
'\\<aleph>' : '\U00002135',
'\\<emptyset>' : '\U00002205',
'\\<nabla>' : '\U00002207',
'\\<partial>' : '\U00002202',
'\\<flat>' : '\U0000266d',
'\\<natural>' : '\U0000266e',
'\\<sharp>' : '\U0000266f',
'\\<angle>' : '\U00002220',
'\\<copyright>' : '\U000000a9',
'\\<registered>' : '\U000000ae',
'\\<hyphen>' : '\U000000ad',
'\\<inverse>' : '\U000000af',
'\\<onequarter>' : '\U000000bc',
'\\<onehalf>' : '\U000000bd',
'\\<threequarters>' : '\U000000be',
'\\<ordfeminine>' : '\U000000aa',
'\\<ordmasculine>' : '\U000000ba',
'\\<section>' : '\U000000a7',
'\\<paragraph>' : '\U000000b6',
'\\<exclamdown>' : '\U000000a1',
'\\<questiondown>' : '\U000000bf',
'\\<euro>' : '\U000020ac',
'\\<pounds>' : '\U000000a3',
'\\<yen>' : '\U000000a5',
'\\<cent>' : '\U000000a2',
'\\<currency>' : '\U000000a4',
'\\<degree>' : '\U000000b0',
'\\<amalg>' : '\U00002a3f',
'\\<mho>' : '\U00002127',
'\\<lozenge>' : '\U000025ca',
'\\<wp>' : '\U00002118',
'\\<wrong>' : '\U00002240',
'\\<struct>' : '\U000022c4',
'\\<acute>' : '\U000000b4',
'\\<index>' : '\U00000131',
'\\<dieresis>' : '\U000000a8',
'\\<cedilla>' : '\U000000b8',
'\\<hungarumlaut>' : '\U000002dd',
'\\<some>' : '\U000003f5',
'\\<newline>' : '\U000023ce',
'\\<open>' : '\U00002039',
'\\<close>' : '\U0000203a',
'\\<here>' : '\U00002302',
'\\<^sub>' : '\U000021e9',
'\\<^sup>' : '\U000021e7',
'\\<^bold>' : '\U00002759',
'\\<^bsub>' : '\U000021d8',
'\\<^esub>' : '\U000021d9',
'\\<^bsup>' : '\U000021d7',
'\\<^esup>' : '\U000021d6',
}
lang_map = {'isabelle' : isabelle_symbols, 'latex' : latex_symbols}
def __init__(self, **options):
Filter.__init__(self, **options)
lang = get_choice_opt(options, 'lang',
['isabelle', 'latex'], 'isabelle')
self.symbols = self.lang_map[lang]
def filter(self, lexer, stream):
for ttype, value in stream:
if value in self.symbols:
yield ttype, self.symbols[value]
else:
yield ttype, value
class KeywordCaseFilter(Filter):
"""Convert keywords to lowercase or uppercase or capitalize them, which
means first letter uppercase, rest lowercase.
This can be useful e.g. if you highlight Pascal code and want to adapt the
code to your styleguide.
Options accepted:
`case` : string
The casing to convert keywords to. Must be one of ``'lower'``,
``'upper'`` or ``'capitalize'``. The default is ``'lower'``.
"""
def __init__(self, **options):
Filter.__init__(self, **options)
case = get_choice_opt(options, 'case',
['lower', 'upper', 'capitalize'], 'lower')
self.convert = getattr(str, case)
def filter(self, lexer, stream):
for ttype, value in stream:
if ttype in Keyword:
yield ttype, self.convert(value)
else:
yield ttype, value
class NameHighlightFilter(Filter):
"""Highlight a normal Name (and Name.*) token with a different token type.
Example::
filter = NameHighlightFilter(
names=['foo', 'bar', 'baz'],
tokentype=Name.Function,
)
This would highlight the names "foo", "bar" and "baz"
as functions. `Name.Function` is the default token type.
Options accepted:
`names` : list of strings
A list of names that should be given the different token type.
There is no default.
`tokentype` : TokenType or string
A token type or a string containing a token type name that is
used for highlighting the strings in `names`. The default is
`Name.Function`.
"""
def __init__(self, **options):
Filter.__init__(self, **options)
self.names = set(get_list_opt(options, 'names', []))
tokentype = options.get('tokentype')
if tokentype:
self.tokentype = string_to_tokentype(tokentype)
else:
self.tokentype = Name.Function
def filter(self, lexer, stream):
for ttype, value in stream:
if ttype in Name and value in self.names:
yield self.tokentype, value
else:
yield ttype, value
class ErrorToken(Exception):
pass
class RaiseOnErrorTokenFilter(Filter):
"""Raise an exception when the lexer generates an error token.
Options accepted:
`excclass` : Exception class
The exception class to raise.
The default is `pygments.filters.ErrorToken`.
.. versionadded:: 0.8
"""
def __init__(self, **options):
Filter.__init__(self, **options)
self.exception = options.get('excclass', ErrorToken)
try:
# issubclass() will raise TypeError if first argument is not a class
if not issubclass(self.exception, Exception):
raise TypeError
except TypeError:
raise OptionError('excclass option is not an exception class')
def filter(self, lexer, stream):
for ttype, value in stream:
if ttype is Error:
raise self.exception(value)
yield ttype, value
class VisibleWhitespaceFilter(Filter):
"""Convert tabs, newlines and/or spaces to visible characters.
Options accepted:
`spaces` : string or bool
If this is a one-character string, spaces will be replaces by this string.
If it is another true value, spaces will be replaced by ``·`` (unicode
MIDDLE DOT). If it is a false value, spaces will not be replaced. The
default is ``False``.
`tabs` : string or bool
The same as for `spaces`, but the default replacement character is ``»``
(unicode RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK). The default value
is ``False``. Note: this will not work if the `tabsize` option for the
lexer is nonzero, as tabs will already have been expanded then.
`tabsize` : int
If tabs are to be replaced by this filter (see the `tabs` option), this
is the total number of characters that a tab should be expanded to.
The default is ``8``.
`newlines` : string or bool
The same as for `spaces`, but the default replacement character is ````
(unicode PILCROW SIGN). The default value is ``False``.
`wstokentype` : bool
If true, give whitespace the special `Whitespace` token type. This allows
styling the visible whitespace differently (e.g. greyed out), but it can
disrupt background colors. The default is ``True``.
.. versionadded:: 0.8
"""
def __init__(self, **options):
Filter.__init__(self, **options)
for name, default in [('spaces', '·'),
('tabs', '»'),
('newlines', '')]:
opt = options.get(name, False)
if isinstance(opt, str) and len(opt) == 1:
setattr(self, name, opt)
else:
setattr(self, name, (opt and default or ''))
tabsize = get_int_opt(options, 'tabsize', 8)
if self.tabs:
self.tabs += ' ' * (tabsize - 1)
if self.newlines:
self.newlines += '\n'
self.wstt = get_bool_opt(options, 'wstokentype', True)
def filter(self, lexer, stream):
if self.wstt:
spaces = self.spaces or ' '
tabs = self.tabs or '\t'
newlines = self.newlines or '\n'
regex = re.compile(r'\s')
def replacefunc(wschar):
if wschar == ' ':
return spaces
elif wschar == '\t':
return tabs
elif wschar == '\n':
return newlines
return wschar
for ttype, value in stream:
yield from _replace_special(ttype, value, regex, Whitespace,
replacefunc)
else:
spaces, tabs, newlines = self.spaces, self.tabs, self.newlines
# simpler processing
for ttype, value in stream:
if spaces:
value = value.replace(' ', spaces)
if tabs:
value = value.replace('\t', tabs)
if newlines:
value = value.replace('\n', newlines)
yield ttype, value
class GobbleFilter(Filter):
"""Gobbles source code lines (eats initial characters).
This filter drops the first ``n`` characters off every line of code. This
may be useful when the source code fed to the lexer is indented by a fixed
amount of space that isn't desired in the output.
Options accepted:
`n` : int
The number of characters to gobble.
.. versionadded:: 1.2
"""
def __init__(self, **options):
Filter.__init__(self, **options)
self.n = get_int_opt(options, 'n', 0)
def gobble(self, value, left):
if left < len(value):
return value[left:], 0
else:
return '', left - len(value)
def filter(self, lexer, stream):
n = self.n
left = n # How many characters left to gobble.
for ttype, value in stream:
# Remove ``left`` tokens from first line, ``n`` from all others.
parts = value.split('\n')
(parts[0], left) = self.gobble(parts[0], left)
for i in range(1, len(parts)):
(parts[i], left) = self.gobble(parts[i], n)
value = '\n'.join(parts)
if value != '':
yield ttype, value
class TokenMergeFilter(Filter):
"""Merges consecutive tokens with the same token type in the output
stream of a lexer.
.. versionadded:: 1.2
"""
def __init__(self, **options):
Filter.__init__(self, **options)
def filter(self, lexer, stream):
current_type = None
current_value = None
for ttype, value in stream:
if ttype is current_type:
current_value += value
else:
if current_type is not None:
yield current_type, current_value
current_type = ttype
current_value = value
if current_type is not None:
yield current_type, current_value
FILTERS = {
'codetagify': CodeTagFilter,
'keywordcase': KeywordCaseFilter,
'highlight': NameHighlightFilter,
'raiseonerror': RaiseOnErrorTokenFilter,
'whitespace': VisibleWhitespaceFilter,
'gobble': GobbleFilter,
'tokenmerge': TokenMergeFilter,
'symbols': SymbolFilter,
}
@@ -1,94 +0,0 @@
"""
pygments.formatter
~~~~~~~~~~~~~~~~~~
Base formatter class.
:copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import codecs
from pip._vendor.pygments.util import get_bool_opt
from pip._vendor.pygments.styles import get_style_by_name
__all__ = ['Formatter']
def _lookup_style(style):
if isinstance(style, str):
return get_style_by_name(style)
return style
class Formatter:
"""
Converts a token stream to text.
Options accepted:
``style``
The style to use, can be a string or a Style subclass
(default: "default"). Not used by e.g. the
TerminalFormatter.
``full``
Tells the formatter to output a "full" document, i.e.
a complete self-contained document. This doesn't have
any effect for some formatters (default: false).
``title``
If ``full`` is true, the title that should be used to
caption the document (default: '').
``encoding``
If given, must be an encoding name. This will be used to
convert the Unicode token strings to byte strings in the
output. If it is "" or None, Unicode strings will be written
to the output file, which most file-like objects do not
support (default: None).
``outencoding``
Overrides ``encoding`` if given.
"""
#: Name of the formatter
name = None
#: Shortcuts for the formatter
aliases = []
#: fn match rules
filenames = []
#: If True, this formatter outputs Unicode strings when no encoding
#: option is given.
unicodeoutput = True
def __init__(self, **options):
self.style = _lookup_style(options.get('style', 'default'))
self.full = get_bool_opt(options, 'full', False)
self.title = options.get('title', '')
self.encoding = options.get('encoding', None) or None
if self.encoding in ('guess', 'chardet'):
# can happen for e.g. pygmentize -O encoding=guess
self.encoding = 'utf-8'
self.encoding = options.get('outencoding') or self.encoding
self.options = options
def get_style_defs(self, arg=''):
"""
Return the style definitions for the current style as a string.
``arg`` is an additional argument whose meaning depends on the
formatter used. Note that ``arg`` can also be a list or tuple
for some formatters like the html formatter.
"""
return ''
def format(self, tokensource, outfile):
"""
Format ``tokensource``, an iterable of ``(tokentype, tokenstring)``
tuples and write it into ``outfile``.
"""
if self.encoding:
# wrap the outfile in a StreamWriter
outfile = codecs.lookup(self.encoding)[3](outfile)
return self.format_unencoded(tokensource, outfile)
@@ -1,153 +0,0 @@
"""
pygments.formatters
~~~~~~~~~~~~~~~~~~~
Pygments formatters.
:copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
import sys
import types
import fnmatch
from os.path import basename
from pip._vendor.pygments.formatters._mapping import FORMATTERS
from pip._vendor.pygments.plugin import find_plugin_formatters
from pip._vendor.pygments.util import ClassNotFound
__all__ = ['get_formatter_by_name', 'get_formatter_for_filename',
'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS)
_formatter_cache = {} # classes by name
_pattern_cache = {}
def _fn_matches(fn, glob):
"""Return whether the supplied file name fn matches pattern filename."""
if glob not in _pattern_cache:
pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
return pattern.match(fn)
return _pattern_cache[glob].match(fn)
def _load_formatters(module_name):
"""Load a formatter (and all others in the module too)."""
mod = __import__(module_name, None, None, ['__all__'])
for formatter_name in mod.__all__:
cls = getattr(mod, formatter_name)
_formatter_cache[cls.name] = cls
def get_all_formatters():
"""Return a generator for all formatter classes."""
# NB: this returns formatter classes, not info like get_all_lexers().
for info in FORMATTERS.values():
if info[1] not in _formatter_cache:
_load_formatters(info[0])
yield _formatter_cache[info[1]]
for _, formatter in find_plugin_formatters():
yield formatter
def find_formatter_class(alias):
"""Lookup a formatter by alias.
Returns None if not found.
"""
for module_name, name, aliases, _, _ in FORMATTERS.values():
if alias in aliases:
if name not in _formatter_cache:
_load_formatters(module_name)
return _formatter_cache[name]
for _, cls in find_plugin_formatters():
if alias in cls.aliases:
return cls
def get_formatter_by_name(_alias, **options):
"""Lookup and instantiate a formatter by alias.
Raises ClassNotFound if not found.
"""
cls = find_formatter_class(_alias)
if cls is None:
raise ClassNotFound("no formatter found for name %r" % _alias)
return cls(**options)
def load_formatter_from_file(filename, formattername="CustomFormatter",
**options):
"""Load a formatter from a file.
This method expects a file located relative to the current working
directory, which contains a class named CustomFormatter. By default,
it expects the Formatter to be named CustomFormatter; you can specify
your own class name as the second argument to this function.
Users should be very careful with the input, because this method
is equivalent to running eval on the input file.
Raises ClassNotFound if there are any problems importing the Formatter.
.. versionadded:: 2.2
"""
try:
# This empty dict will contain the namespace for the exec'd file
custom_namespace = {}
with open(filename, 'rb') as f:
exec(f.read(), custom_namespace)
# Retrieve the class `formattername` from that namespace
if formattername not in custom_namespace:
raise ClassNotFound('no valid %s class found in %s' %
(formattername, filename))
formatter_class = custom_namespace[formattername]
# And finally instantiate it with the options
return formatter_class(**options)
except OSError as err:
raise ClassNotFound('cannot read %s: %s' % (filename, err))
except ClassNotFound:
raise
except Exception as err:
raise ClassNotFound('error when loading custom formatter: %s' % err)
def get_formatter_for_filename(fn, **options):
"""Lookup and instantiate a formatter by filename pattern.
Raises ClassNotFound if not found.
"""
fn = basename(fn)
for modname, name, _, filenames, _ in FORMATTERS.values():
for filename in filenames:
if _fn_matches(fn, filename):
if name not in _formatter_cache:
_load_formatters(modname)
return _formatter_cache[name](**options)
for cls in find_plugin_formatters():
for filename in cls.filenames:
if _fn_matches(fn, filename):
return cls(**options)
raise ClassNotFound("no formatter found for file name %r" % fn)
class _automodule(types.ModuleType):
"""Automatically import formatters."""
def __getattr__(self, name):
info = FORMATTERS.get(name)
if info:
_load_formatters(info[0])
cls = _formatter_cache[info[1]]
setattr(self, name, cls)
return cls
raise AttributeError(name)
oldmod = sys.modules[__name__]
newmod = _automodule(__name__)
newmod.__dict__.update(oldmod.__dict__)
sys.modules[__name__] = newmod
del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types

Some files were not shown because too many files have changed in this diff Show More