更改enroll命名,添加了注释,向get_error_msg中添加了一些错误代码

This commit is contained in:
ygm1881
2022-05-05 22:59:35 +08:00
parent 51b5e374a3
commit ece69eaf57
4637 changed files with 7699 additions and 608140 deletions
+83 -28
View File
@@ -1,15 +1,15 @@
"""Extensions to the 'distutils' for large or complex distributions"""
from fnmatch import fnmatchcase
import functools
import os
import re
import warnings
import _distutils_hack.override # noqa: F401
import distutils.core
from distutils.errors import DistutilsOptionError
from distutils.util import convert_path as _convert_path
from distutils.util import convert_path
from ._deprecation_warning import SetuptoolsDeprecationWarning
@@ -17,7 +17,6 @@ import setuptools.version
from setuptools.extension import Extension
from setuptools.dist import Distribution
from setuptools.depends import Require
from setuptools.discovery import PackageFinder, PEP420PackageFinder
from . import monkey
from . import logging
@@ -38,6 +37,85 @@ __version__ = setuptools.version.__version__
bootstrap_install_from = None
class PackageFinder:
"""
Generate a list of all Python packages found within a directory
"""
@classmethod
def find(cls, where='.', exclude=(), include=('*',)):
"""Return a list all Python packages found within directory 'where'
'where' is the root directory which will be searched for packages. It
should be supplied as a "cross-platform" (i.e. URL-style) path; it will
be converted to the appropriate local path syntax.
'exclude' is a sequence of package names to exclude; '*' can be used
as a wildcard in the names, such that 'foo.*' will exclude all
subpackages of 'foo' (but not 'foo' itself).
'include' is a sequence of package names to include. If it's
specified, only the named packages will be included. If it's not
specified, all found packages will be included. 'include' can contain
shell style wildcard patterns just like 'exclude'.
"""
return list(
cls._find_packages_iter(
convert_path(where),
cls._build_filter('ez_setup', '*__pycache__', *exclude),
cls._build_filter(*include),
)
)
@classmethod
def _find_packages_iter(cls, where, exclude, include):
"""
All the packages found in 'where' that pass the 'include' filter, but
not the 'exclude' filter.
"""
for root, dirs, files in os.walk(where, followlinks=True):
# Copy dirs to iterate over it, then empty dirs.
all_dirs = dirs[:]
dirs[:] = []
for dir in all_dirs:
full_path = os.path.join(root, dir)
rel_path = os.path.relpath(full_path, where)
package = rel_path.replace(os.path.sep, '.')
# Skip directory trees that are not valid packages
if '.' in dir or not cls._looks_like_package(full_path):
continue
# Should this package be included?
if include(package) and not exclude(package):
yield package
# Keep searching subdirectories, as there may be more packages
# down there, even if the parent was excluded.
dirs.append(dir)
@staticmethod
def _looks_like_package(path):
"""Does a directory look like a package?"""
return os.path.isfile(os.path.join(path, '__init__.py'))
@staticmethod
def _build_filter(*patterns):
"""
Given a list of patterns, return a callable that will be true only if
the input matches at least one of the patterns.
"""
return lambda name: any(fnmatchcase(name, pat=pat) for pat in patterns)
class PEP420PackageFinder(PackageFinder):
@staticmethod
def _looks_like_package(path):
return True
find_packages = PackageFinder.find
find_namespace_packages = PEP420PackageFinder.find
@@ -54,17 +132,7 @@ def _install_setup_requires(attrs):
def __init__(self, attrs):
_incl = 'dependency_links', 'setup_requires'
filtered = {k: attrs[k] for k in set(_incl) & set(attrs)}
super().__init__(filtered)
# Prevent accidentally triggering discovery with incomplete set of attrs
self.set_defaults._disable()
def _get_project_config_files(self, filenames=None):
"""Ignore ``pyproject.toml``, they are not related to setup_requires"""
try:
cfg, toml = super()._split_standard_project_metadata(filenames)
return cfg, ()
except Exception:
return filenames, ()
distutils.core.Distribution.__init__(self, filtered)
def finalize_options(self):
"""
@@ -103,7 +171,7 @@ class Command(_Command):
Construct the command for dist, updating
vars(self) with any keyword parameters.
"""
super().__init__(dist)
_Command.__init__(self, dist)
vars(self).update(kw)
def _ensure_stringlike(self, option, what, default=None):
@@ -168,19 +236,6 @@ def findall(dir=os.curdir):
return list(files)
@functools.wraps(_convert_path)
def convert_path(pathname):
from inspect import cleandoc
msg = """
The function `convert_path` is considered internal and not part of the public API.
Its direct usage by 3rd-party packages is considered deprecated and the function
may be removed in the future.
"""
warnings.warn(cleandoc(msg), SetuptoolsDeprecationWarning)
return _convert_path(pathname)
class sic(str):
"""Treat this string as-is (https://en.wikipedia.org/wiki/Sic)"""
@@ -1,56 +0,0 @@
import collections
import itertools
# from jaraco.collections 3.5.1
class DictStack(list, collections.abc.Mapping):
"""
A stack of dictionaries that behaves as a view on those dictionaries,
giving preference to the last.
>>> stack = DictStack([dict(a=1, c=2), dict(b=2, a=2)])
>>> stack['a']
2
>>> stack['b']
2
>>> stack['c']
2
>>> len(stack)
3
>>> stack.push(dict(a=3))
>>> stack['a']
3
>>> set(stack.keys()) == set(['a', 'b', 'c'])
True
>>> set(stack.items()) == set([('a', 3), ('b', 2), ('c', 2)])
True
>>> dict(**stack) == dict(stack) == dict(a=3, c=2, b=2)
True
>>> d = stack.pop()
>>> stack['a']
2
>>> d = stack.pop()
>>> stack['a']
1
>>> stack.get('b', None)
>>> 'c' in stack
True
"""
def __iter__(self):
dicts = list.__iter__(self)
return iter(set(itertools.chain.from_iterable(c.keys() for c in dicts)))
def __getitem__(self, key):
for scope in reversed(tuple(list.__iter__(self))):
if key in scope:
return scope[key]
raise KeyError(key)
push = list.append
def __contains__(self, other):
return collections.abc.Mapping.__contains__(self, other)
def __len__(self):
return len(list(iter(self)))
@@ -1,12 +0,0 @@
import sys
import importlib
def bypass_compiler_fixup(cmd, args):
return cmd
if sys.platform == 'darwin':
compiler_fixup = importlib.import_module('_osx_support').compiler_fixup
else:
compiler_fixup = bypass_compiler_fixup
@@ -203,7 +203,7 @@ class MSVCCompiler(CCompiler) :
def __init__(self, verbose=0, dry_run=0, force=0):
super().__init__(verbose, dry_run, force)
CCompiler.__init__ (self, verbose, dry_run, force)
# target platform (.plat_name is consistent with 'bdist')
self.plat_name = None
self.initialized = False
@@ -55,7 +55,7 @@ class BCPPCompiler(CCompiler) :
dry_run=0,
force=0):
super().__init__(verbose, dry_run, force)
CCompiler.__init__ (self, verbose, dry_run, force)
# These executables are assumed to all be in the path.
# Borland doesn't seem to use any special registry settings to
@@ -27,7 +27,7 @@ class PyDialog(Dialog):
def __init__(self, *args, **kw):
"""Dialog(database, name, x, y, w, h, attributes, title, first,
default, cancel, bitmap=true)"""
super().__init__(*args)
Dialog.__init__(self, *args)
ruler = self.h - 36
bmwidth = 152*ruler/328
#if kw.get("bitmap", True):
@@ -81,8 +81,7 @@ class build(Command):
"--plat-name only supported on Windows (try "
"using './configure --help' on your platform)")
plat_specifier = ".%s-%s" % (self.plat_name,
sys.implementation.cache_tag)
plat_specifier = ".%s-%d.%d" % (self.plat_name, *sys.version_info[:2])
# Make it so Python 2.x and Python 2.x with --with-pydebug don't
# share the same build directories. Doing so confuses the build
@@ -2,8 +2,7 @@
Implements the Distutils 'build_scripts' command."""
import os
import re
import os, re
from stat import ST_MODE
from distutils import sysconfig
from distutils.core import Command
@@ -12,14 +11,8 @@ from distutils.util import convert_path
from distutils import log
import tokenize
shebang_pattern = re.compile('^#!.*python[0-9.]*([ \t].*)?$')
"""
Pattern matching a Python interpreter indicated in first line of a script.
"""
# for Setuptools compatibility
first_line_re = shebang_pattern
# check if Python is called on the first line with this expression
first_line_re = re.compile(b'^#!.*python[0-9.]*([ \t].*)?$')
class build_scripts(Command):
@@ -33,11 +26,13 @@ class build_scripts(Command):
boolean_options = ['force']
def initialize_options(self):
self.build_dir = None
self.scripts = None
self.force = None
self.executable = None
self.outfiles = None
def finalize_options(self):
self.set_undefined_options('build',
@@ -54,117 +49,104 @@ class build_scripts(Command):
return
self.copy_scripts()
def copy_scripts(self):
"""
Copy each script listed in ``self.scripts``.
If a script is marked as a Python script (first line matches
'shebang_pattern', i.e. starts with ``#!`` and contains
"python"), then adjust in the copy the first line to refer to
the current Python interpreter.
def copy_scripts(self):
r"""Copy each script listed in 'self.scripts'; if it's marked as a
Python script in the Unix way (first line matches 'first_line_re',
ie. starts with "\#!" and contains "python"), then adjust the first
line to refer to the current Python interpreter as we copy.
"""
self.mkpath(self.build_dir)
outfiles = []
updated_files = []
for script in self.scripts:
self._copy_script(script, outfiles, updated_files)
adjust = False
script = convert_path(script)
outfile = os.path.join(self.build_dir, os.path.basename(script))
outfiles.append(outfile)
self._change_modes(outfiles)
if not self.force and not newer(script, outfile):
log.debug("not copying %s (up-to-date)", script)
continue
return outfiles, updated_files
# Always open the file, but ignore failures in dry-run mode --
# that way, we'll get accurate feedback if we can read the
# script.
try:
f = open(script, "rb")
except OSError:
if not self.dry_run:
raise
f = None
else:
encoding, lines = tokenize.detect_encoding(f.readline)
f.seek(0)
first_line = f.readline()
if not first_line:
self.warn("%s is an empty file (skipping)" % script)
continue
def _copy_script(self, script, outfiles, updated_files):
shebang_match = None
script = convert_path(script)
outfile = os.path.join(self.build_dir, os.path.basename(script))
outfiles.append(outfile)
match = first_line_re.match(first_line)
if match:
adjust = True
post_interp = match.group(1) or b''
if not self.force and not newer(script, outfile):
log.debug("not copying %s (up-to-date)", script)
return
if adjust:
log.info("copying and adjusting %s -> %s", script,
self.build_dir)
updated_files.append(outfile)
if not self.dry_run:
if not sysconfig.python_build:
executable = self.executable
else:
executable = os.path.join(
sysconfig.get_config_var("BINDIR"),
"python%s%s" % (sysconfig.get_config_var("VERSION"),
sysconfig.get_config_var("EXE")))
executable = os.fsencode(executable)
shebang = b"#!" + executable + post_interp + b"\n"
# Python parser starts to read a script using UTF-8 until
# it gets a #coding:xxx cookie. The shebang has to be the
# first line of a file, the #coding:xxx cookie cannot be
# written before. So the shebang has to be decodable from
# UTF-8.
try:
shebang.decode('utf-8')
except UnicodeDecodeError:
raise ValueError(
"The shebang ({!r}) is not decodable "
"from utf-8".format(shebang))
# If the script is encoded to a custom encoding (use a
# #coding:xxx cookie), the shebang has to be decodable from
# the script encoding too.
try:
shebang.decode(encoding)
except UnicodeDecodeError:
raise ValueError(
"The shebang ({!r}) is not decodable "
"from the script encoding ({})"
.format(shebang, encoding))
with open(outfile, "wb") as outf:
outf.write(shebang)
outf.writelines(f.readlines())
if f:
f.close()
else:
if f:
f.close()
updated_files.append(outfile)
self.copy_file(script, outfile)
# Always open the file, but ignore failures in dry-run mode
# in order to attempt to copy directly.
try:
f = tokenize.open(script)
except OSError:
if not self.dry_run:
raise
f = None
else:
first_line = f.readline()
if not first_line:
self.warn("%s is an empty file (skipping)" % script)
return
shebang_match = shebang_pattern.match(first_line)
updated_files.append(outfile)
if shebang_match:
log.info("copying and adjusting %s -> %s", script,
self.build_dir)
if not self.dry_run:
if not sysconfig.python_build:
executable = self.executable
if os.name == 'posix':
for file in outfiles:
if self.dry_run:
log.info("changing mode of %s", file)
else:
executable = os.path.join(
sysconfig.get_config_var("BINDIR"),
"python%s%s" % (
sysconfig.get_config_var("VERSION"),
sysconfig.get_config_var("EXE")))
post_interp = shebang_match.group(1) or ''
shebang = "#!" + executable + post_interp + "\n"
self._validate_shebang(shebang, f.encoding)
with open(outfile, "w", encoding=f.encoding) as outf:
outf.write(shebang)
outf.writelines(f.readlines())
if f:
f.close()
else:
if f:
f.close()
self.copy_file(script, outfile)
def _change_modes(self, outfiles):
if os.name != 'posix':
return
for file in outfiles:
self._change_mode(file)
def _change_mode(self, file):
if self.dry_run:
log.info("changing mode of %s", file)
return
oldmode = os.stat(file)[ST_MODE] & 0o7777
newmode = (oldmode | 0o555) & 0o7777
if newmode != oldmode:
log.info("changing mode of %s from %o to %o",
file, oldmode, newmode)
os.chmod(file, newmode)
@staticmethod
def _validate_shebang(shebang, encoding):
# Python parser starts to read a script using UTF-8 until
# it gets a #coding:xxx cookie. The shebang has to be the
# first line of a file, the #coding:xxx cookie cannot be
# written before. So the shebang has to be encodable to
# UTF-8.
try:
shebang.encode('utf-8')
except UnicodeEncodeError:
raise ValueError(
"The shebang ({!r}) is not encodable "
"to utf-8".format(shebang))
# If the script is encoded to a custom encoding (use a
# #coding:xxx cookie), the shebang has to be encodable to
# the script encoding too.
try:
shebang.encode(encoding)
except UnicodeEncodeError:
raise ValueError(
"The shebang ({!r}) is not encodable "
"to the script encoding ({})"
.format(shebang, encoding))
oldmode = os.stat(file)[ST_MODE] & 0o7777
newmode = (oldmode | 0o555) & 0o7777
if newmode != oldmode:
log.info("changing mode of %s from %o to %o",
file, oldmode, newmode)
os.chmod(file, newmode)
# XXX should we modify self.outfiles?
return outfiles, updated_files
@@ -2,8 +2,6 @@
Implements the Distutils 'check' command.
"""
from email.utils import getaddresses
from distutils.core import Command
from distutils.errors import DistutilsSetupError
@@ -19,7 +17,7 @@ try:
def __init__(self, source, report_level, halt_level, stream=None,
debug=0, encoding='ascii', error_handler='replace'):
self.messages = []
super().__init__(source, report_level, halt_level, stream,
Reporter.__init__(self, source, report_level, halt_level, stream,
debug, encoding, error_handler)
def system_message(self, level, message, *children, **kwargs):
@@ -98,39 +96,19 @@ class check(Command):
if missing:
self.warn("missing required meta-data: %s" % ', '.join(missing))
if not (
self._check_contact("author", metadata) or
self._check_contact("maintainer", metadata)
):
if metadata.author:
if not metadata.author_email:
self.warn("missing meta-data: if 'author' supplied, " +
"'author_email' should be supplied too")
elif metadata.maintainer:
if not metadata.maintainer_email:
self.warn("missing meta-data: if 'maintainer' supplied, " +
"'maintainer_email' should be supplied too")
else:
self.warn("missing meta-data: either (author and author_email) " +
"or (maintainer and maintainer_email) " +
"should be supplied")
def _check_contact(self, kind, metadata):
"""
Returns True if the contact's name is specified and False otherwise.
This function will warn if the contact's email is not specified.
"""
name = getattr(metadata, kind) or ''
email = getattr(metadata, kind + '_email') or ''
msg = ("missing meta-data: if '{}' supplied, " +
"'{}' should be supplied too")
if name and email:
return True
if name:
self.warn(msg.format(kind, kind + '_email'))
return True
addresses = [(alias, addr) for alias, addr in getaddresses([email])]
if any(alias and addr for alias, addr in addresses):
# The contact's name can be encoded in the email: `Name <email>`
return True
return False
def check_restructuredtext(self):
"""Checks if the long string fields are reST-compliant."""
data = self.distribution.get_long_description()
@@ -17,7 +17,6 @@ from distutils.file_util import write_file
from distutils.util import convert_path, subst_vars, change_root
from distutils.util import get_platform
from distutils.errors import DistutilsOptionError
from .. import _collections
from site import USER_BASE
from site import USER_SITE
@@ -68,8 +67,8 @@ if HAS_USER_SITE:
INSTALL_SCHEMES['nt_user'] = {
'purelib': '{usersite}',
'platlib': '{usersite}',
'headers': '{userbase}/{implementation}{py_version_nodot_plat}/Include/{dist_name}',
'scripts': '{userbase}/{implementation}{py_version_nodot_plat}/Scripts',
'headers': '{userbase}/{implementation}{py_version_nodot}/Include/{dist_name}',
'scripts': '{userbase}/{implementation}{py_version_nodot}/Scripts',
'data' : '{userbase}',
}
@@ -395,35 +394,26 @@ class install(Command):
except AttributeError:
# sys.abiflags may not be defined on all platforms.
abiflags = ''
local_vars = {
'dist_name': self.distribution.get_name(),
'dist_version': self.distribution.get_version(),
'dist_fullname': self.distribution.get_fullname(),
'py_version': py_version,
'py_version_short': '%d.%d' % sys.version_info[:2],
'py_version_nodot': '%d%d' % sys.version_info[:2],
'sys_prefix': prefix,
'prefix': prefix,
'sys_exec_prefix': exec_prefix,
'exec_prefix': exec_prefix,
'abiflags': abiflags,
'platlibdir': getattr(sys, 'platlibdir', 'lib'),
'implementation_lower': _get_implementation().lower(),
'implementation': _get_implementation(),
}
# vars for compatibility on older Pythons
compat_vars = dict(
# Python 3.9 and earlier
py_version_nodot_plat=getattr(sys, 'winver', '').replace('.', ''),
)
self.config_vars = {'dist_name': self.distribution.get_name(),
'dist_version': self.distribution.get_version(),
'dist_fullname': self.distribution.get_fullname(),
'py_version': py_version,
'py_version_short': '%d.%d' % sys.version_info[:2],
'py_version_nodot': '%d%d' % sys.version_info[:2],
'sys_prefix': prefix,
'prefix': prefix,
'sys_exec_prefix': exec_prefix,
'exec_prefix': exec_prefix,
'abiflags': abiflags,
'platlibdir': getattr(sys, 'platlibdir', 'lib'),
'implementation_lower': _get_implementation().lower(),
'implementation': _get_implementation(),
'platsubdir': sysconfig.get_config_var('platsubdir'),
}
if HAS_USER_SITE:
local_vars['userbase'] = self.install_userbase
local_vars['usersite'] = self.install_usersite
self.config_vars = _collections.DictStack(
[compat_vars, sysconfig.get_config_vars(), local_vars])
self.config_vars['userbase'] = self.install_userbase
self.config_vars['usersite'] = self.install_usersite
self.expand_basedirs()
@@ -431,13 +421,15 @@ class install(Command):
# Now define config vars for the base directories so we can expand
# everything else.
local_vars['base'] = self.install_base
local_vars['platbase'] = self.install_platbase
self.config_vars['base'] = self.install_base
self.config_vars['platbase'] = self.install_platbase
self.config_vars['installed_base'] = (
sysconfig.get_config_vars()['installed_base'])
if DEBUG:
from pprint import pprint
print("config vars:")
pprint(dict(self.config_vars))
pprint(self.config_vars)
# Expand "~" and configuration variables in the installation
# directories.
@@ -108,7 +108,7 @@ class CygwinCCompiler(UnixCCompiler):
def __init__(self, verbose=0, dry_run=0, force=0):
super().__init__(verbose, dry_run, force)
UnixCCompiler.__init__(self, verbose, dry_run, force)
status, details = check_config_h()
self.debug_print("Python's GCC status: %s (details: %s)" %
@@ -268,7 +268,7 @@ class Mingw32CCompiler(CygwinCCompiler):
def __init__(self, verbose=0, dry_run=0, force=0):
super().__init__ (verbose, dry_run, force)
CygwinCCompiler.__init__ (self, verbose, dry_run, force)
shared_option = "-shared"
@@ -324,7 +324,7 @@ class MSVCCompiler(CCompiler) :
exe_extension = '.exe'
def __init__(self, verbose=0, dry_run=0, force=0):
super().__init__(verbose, dry_run, force)
CCompiler.__init__ (self, verbose, dry_run, force)
self.__version = VERSION
self.__root = r"Software\Microsoft\VisualStudio"
# self.__macros = MACROS
@@ -228,7 +228,7 @@ class MSVCCompiler(CCompiler) :
exe_extension = '.exe'
def __init__(self, verbose=0, dry_run=0, force=0):
super().__init__(verbose, dry_run, force)
CCompiler.__init__ (self, verbose, dry_run, force)
self.__version = get_build_version()
self.__arch = get_build_architecture()
if self.__arch == "Intel":
@@ -1,21 +0,0 @@
import sys
import platform
def add_ext_suffix_39(vars):
"""
Ensure vars contains 'EXT_SUFFIX'. pypa/distutils#130
"""
import _imp
ext_suffix = _imp.extension_suffixes()[0]
vars.update(
EXT_SUFFIX=ext_suffix,
# sysconfig sets SO to match EXT_SUFFIX, so maintain
# that expectation.
# https://github.com/python/cpython/blob/785cc6770588de087d09e89a69110af2542be208/Lib/sysconfig.py#L671-L673
SO=ext_suffix,
)
needs_ext_suffix = sys.version_info < (3, 10) and platform.system() == 'Windows'
add_ext_suffix = add_ext_suffix_39 if needs_ext_suffix else lambda vars: None
@@ -10,7 +10,7 @@ import sys
import os
import subprocess
from distutils.errors import DistutilsExecError
from distutils.errors import DistutilsPlatformError, DistutilsExecError
from distutils.debug import DEBUG
from distutils import log
@@ -9,13 +9,13 @@ Written by: Fred L. Drake, Jr.
Email: <fdrake@acm.org>
"""
import _imp
import os
import re
import sys
import sysconfig
from .errors import DistutilsPlatformError
from . import py39compat
IS_PYPY = '__pypy__' in sys.builtin_module_names
@@ -48,7 +48,6 @@ def _is_python_source_dir(d):
return True
return False
_sys_home = getattr(sys, '_home', None)
if os.name == 'nt':
@@ -60,13 +59,11 @@ if os.name == 'nt':
project_base = _fix_pcbuild(project_base)
_sys_home = _fix_pcbuild(_sys_home)
def _python_build():
if _sys_home:
return _is_python_source_dir(_sys_home)
return _is_python_source_dir(project_base)
python_build = _python_build()
@@ -82,7 +79,6 @@ except AttributeError:
# this attribute, which is fine.
pass
def get_python_version():
"""Return a string containing the major and minor Python version,
leaving off the patchlevel. Sample return values could be '1.5'
@@ -196,6 +192,7 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
"on platform '%s'" % os.name)
def customize_compiler(compiler):
"""Do any platform-specific customization of a CCompiler instance.
@@ -220,9 +217,8 @@ def customize_compiler(compiler):
_config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
(cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \
get_config_vars(
'CC', 'CXX', 'CFLAGS',
'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS')
get_config_vars('CC', 'CXX', 'CFLAGS',
'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS')
if 'CC' in os.environ:
newcc = os.environ['CC']
@@ -284,6 +280,7 @@ def get_config_h_filename():
return sysconfig.get_config_h_filename()
def get_makefile_filename():
"""Return full pathname of installed Makefile from the Python build."""
return sysconfig.get_makefile_filename()
@@ -305,7 +302,6 @@ _variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
def parse_makefile(fn, g=None):
"""Parse a Makefile-style file.
@@ -314,9 +310,7 @@ def parse_makefile(fn, g=None):
used instead of a new dictionary.
"""
from distutils.text_file import TextFile
fp = TextFile(
fn, strip_comments=1, skip_blanks=1, join_lines=1,
errors="surrogateescape")
fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1, errors="surrogateescape")
if g is None:
g = {}
@@ -325,7 +319,7 @@ def parse_makefile(fn, g=None):
while True:
line = fp.readline()
if line is None: # eof
if line is None: # eof
break
m = _variable_rx.match(line)
if m:
@@ -369,8 +363,7 @@ def parse_makefile(fn, g=None):
item = os.environ[n]
elif n in renamed_variables:
if name.startswith('PY_') and \
name[3:] in renamed_variables:
if name.startswith('PY_') and name[3:] in renamed_variables:
item = ""
elif 'PY_' + n in notdone:
@@ -386,8 +379,7 @@ def parse_makefile(fn, g=None):
if "$" in after:
notdone[name] = value
else:
try:
value = int(value)
try: value = int(value)
except ValueError:
done[name] = value.strip()
else:
@@ -395,7 +387,7 @@ def parse_makefile(fn, g=None):
del notdone[name]
if name.startswith('PY_') \
and name[3:] in renamed_variables:
and name[3:] in renamed_variables:
name = name[3:]
if name not in done:
@@ -444,6 +436,51 @@ def expand_makefile_vars(s, vars):
_config_vars = None
_sysconfig_name_tmpl = '_sysconfigdata_{abi}_{platform}_{multiarch}'
def _init_posix():
"""Initialize the module as appropriate for POSIX systems."""
# _sysconfigdata is generated at build time, see the sysconfig module
name = os.environ.get(
'_PYTHON_SYSCONFIGDATA_NAME',
_sysconfig_name_tmpl.format(
abi=sys.abiflags,
platform=sys.platform,
multiarch=getattr(sys.implementation, '_multiarch', ''),
),
)
try:
_temp = __import__(name, globals(), locals(), ['build_time_vars'], 0)
except ImportError:
# Python 3.5 and pypy 7.3.1
_temp = __import__(
'_sysconfigdata', globals(), locals(), ['build_time_vars'], 0)
build_time_vars = _temp.build_time_vars
global _config_vars
_config_vars = {}
_config_vars.update(build_time_vars)
def _init_nt():
"""Initialize the module as appropriate for NT"""
g = {}
# set basic install directories
g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
# XXX hmmm.. a normal install puts include files here
g['INCLUDEPY'] = get_python_inc(plat_specific=0)
g['EXT_SUFFIX'] = _imp.extension_suffixes()[0]
g['EXE'] = ".exe"
g['VERSION'] = get_python_version().replace(".", "")
g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
global _config_vars
_config_vars = g
def get_config_vars(*args):
"""With no arguments, return a dictionary of all configuration
variables relevant for the current platform. Generally this includes
@@ -456,8 +493,60 @@ def get_config_vars(*args):
"""
global _config_vars
if _config_vars is None:
_config_vars = sysconfig.get_config_vars().copy()
py39compat.add_ext_suffix(_config_vars)
func = globals().get("_init_" + os.name)
if func:
func()
else:
_config_vars = {}
# Normalized versions of prefix and exec_prefix are handy to have;
# in fact, these are the standard versions used most places in the
# Distutils.
_config_vars['prefix'] = PREFIX
_config_vars['exec_prefix'] = EXEC_PREFIX
if not IS_PYPY:
# For backward compatibility, see issue19555
SO = _config_vars.get('EXT_SUFFIX')
if SO is not None:
_config_vars['SO'] = SO
# Always convert srcdir to an absolute path
srcdir = _config_vars.get('srcdir', project_base)
if os.name == 'posix':
if python_build:
# If srcdir is a relative path (typically '.' or '..')
# then it should be interpreted relative to the directory
# containing Makefile.
base = os.path.dirname(get_makefile_filename())
srcdir = os.path.join(base, srcdir)
else:
# srcdir is not meaningful since the installation is
# spread about the filesystem. We choose the
# directory containing the Makefile since we know it
# exists.
srcdir = os.path.dirname(get_makefile_filename())
_config_vars['srcdir'] = os.path.abspath(os.path.normpath(srcdir))
# Convert srcdir into an absolute path if it appears necessary.
# Normally it is relative to the build directory. However, during
# testing, for example, we might be running a non-installed python
# from a different directory.
if python_build and os.name == "posix":
base = project_base
if (not os.path.isabs(_config_vars['srcdir']) and
base != os.getcwd()):
# srcdir is relative and we are not in the same directory
# as the executable. Assume executable is in the build
# directory and make srcdir absolute.
srcdir = os.path.join(base, _config_vars['srcdir'])
_config_vars['srcdir'] = os.path.normpath(srcdir)
# OS X platforms require special customization to handle
# multi-architecture, multi-os-version installers
if sys.platform == 'darwin':
import _osx_support
_osx_support.customize_config_vars(_config_vars)
if args:
vals = []
@@ -467,7 +556,6 @@ def get_config_vars(*args):
else:
return _config_vars
def get_config_var(name):
"""Return the value of a single variable using the dictionary
returned by 'get_config_vars()'. Equivalent to
@@ -475,6 +563,5 @@ def get_config_var(name):
"""
if name == 'SO':
import warnings
warnings.warn(
'SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2)
warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2)
return get_config_vars().get(name)
@@ -22,7 +22,9 @@ from distutils.ccompiler import \
from distutils.errors import \
DistutilsExecError, CompileError, LibError, LinkError
from distutils import log
from ._macos_compat import compiler_fixup
if sys.platform == 'darwin':
import _osx_support
# XXX Things not currently handled:
# * optimization/debug/warning flags; we just use whatever's in Python's
@@ -40,66 +42,6 @@ from ._macos_compat import compiler_fixup
# options and carry on.
def _split_env(cmd):
"""
For macOS, split command into 'env' portion (if any)
and the rest of the linker command.
>>> _split_env(['a', 'b', 'c'])
([], ['a', 'b', 'c'])
>>> _split_env(['/usr/bin/env', 'A=3', 'gcc'])
(['/usr/bin/env', 'A=3'], ['gcc'])
"""
pivot = 0
if os.path.basename(cmd[0]) == "env":
pivot = 1
while '=' in cmd[pivot]:
pivot += 1
return cmd[:pivot], cmd[pivot:]
def _split_aix(cmd):
"""
AIX platforms prefix the compiler with the ld_so_aix
script, so split that from the linker command.
>>> _split_aix(['a', 'b', 'c'])
([], ['a', 'b', 'c'])
>>> _split_aix(['/bin/foo/ld_so_aix', 'gcc'])
(['/bin/foo/ld_so_aix'], ['gcc'])
"""
pivot = os.path.basename(cmd[0]) == 'ld_so_aix'
return cmd[:pivot], cmd[pivot:]
def _linker_params(linker_cmd, compiler_cmd):
"""
The linker command usually begins with the compiler
command (possibly multiple elements), followed by zero or more
params for shared library building.
If the LDSHARED env variable overrides the linker command,
however, the commands may not match.
Return the best guess of the linker parameters by stripping
the linker command. If the compiler command does not
match the linker command, assume the linker command is
just the first element.
>>> _linker_params('gcc foo bar'.split(), ['gcc'])
['foo', 'bar']
>>> _linker_params('gcc foo bar'.split(), ['other'])
['foo', 'bar']
>>> _linker_params('ccache gcc foo bar'.split(), 'ccache gcc'.split())
['foo', 'bar']
>>> _linker_params(['gcc'], ['gcc'])
[]
"""
c_len = len(compiler_cmd)
pivot = c_len if linker_cmd[:c_len] == compiler_cmd else 1
return linker_cmd[pivot:]
class UnixCCompiler(CCompiler):
compiler_type = 'unix'
@@ -167,8 +109,10 @@ class UnixCCompiler(CCompiler):
raise CompileError(msg)
def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts):
compiler_so = compiler_fixup(
self.compiler_so, cc_args + extra_postargs)
compiler_so = self.compiler_so
if sys.platform == 'darwin':
compiler_so = _osx_support.compiler_fixup(compiler_so,
cc_args + extra_postargs)
try:
self.spawn(compiler_so + cc_args + [src, '-o', obj] +
extra_postargs)
@@ -229,22 +173,33 @@ class UnixCCompiler(CCompiler):
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
# Select a linker based on context: linker_exe when
# building an executable or linker_so (with shared options)
# when building a shared library.
building_exe = target_desc == CCompiler.EXECUTABLE
linker = (self.linker_exe if building_exe else self.linker_so)[:]
if target_desc == CCompiler.EXECUTABLE:
linker = self.linker_exe[:]
else:
linker = self.linker_so[:]
if target_lang == "c++" and self.compiler_cxx:
env, linker_ne = _split_env(linker)
aix, linker_na = _split_aix(linker_ne)
_, compiler_cxx_ne = _split_env(self.compiler_cxx)
_, linker_exe_ne = _split_env(self.linker_exe)
# skip over environment variable settings if /usr/bin/env
# is used to set up the linker's environment.
# This is needed on OSX. Note: this assumes that the
# normal and C++ compiler have the same environment
# settings.
i = 0
if os.path.basename(linker[0]) == "env":
i = 1
while '=' in linker[i]:
i += 1
params = _linker_params(linker_na, linker_exe_ne)
linker = env + aix + compiler_cxx_ne + params
if os.path.basename(linker[i]) == 'ld_so_aix':
# AIX platforms prefix the compiler with the ld_so_aix
# script, so we need to adjust our linker index
offset = 1
else:
offset = 0
linker = compiler_fixup(linker, ld_args)
linker[i+offset] = self.compiler_cxx[i]
if sys.platform == 'darwin':
linker = _osx_support.compiler_fixup(linker, ld_args)
self.spawn(linker + ld_args)
except DistutilsExecError as msg:
@@ -9,7 +9,6 @@ import re
import importlib.util
import string
import sys
import sysconfig
from distutils.errors import DistutilsPlatformError
from distutils.dep_util import newer
from distutils.spawn import spawn
@@ -21,29 +20,82 @@ from .py35compat import _optim_args_from_interpreter_flags
def get_host_platform():
"""Return a string that identifies the current platform. This is used mainly to
distinguish platform-specific build directories and platform-specific built
distributions.
distributions. Typically includes the OS name and version and the
architecture (as supplied by 'os.uname()'), although the exact information
included depends on the OS; eg. on Linux, the kernel version isn't
particularly important.
Examples of returned values:
linux-i586
linux-alpha (?)
solaris-2.6-sun4u
Windows will return one of:
win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
win32 (all others - specifically, sys.platform is returned)
For other non-POSIX platforms, currently just returns 'sys.platform'.
"""
if os.name == 'nt':
if 'amd64' in sys.version.lower():
return 'win-amd64'
if '(arm)' in sys.version.lower():
return 'win-arm32'
if '(arm64)' in sys.version.lower():
return 'win-arm64'
return sys.platform
# We initially exposed platforms as defined in Python 3.9
# even with older Python versions when distutils was split out.
# Now that we delegate to stdlib sysconfig we need to restore this
# in case anyone has started to depend on it.
# Set for cross builds explicitly
if "_PYTHON_HOST_PLATFORM" in os.environ:
return os.environ["_PYTHON_HOST_PLATFORM"]
if sys.version_info < (3, 8):
if os.name == 'nt':
if '(arm)' in sys.version.lower():
return 'win-arm32'
if '(arm64)' in sys.version.lower():
return 'win-arm64'
if os.name != "posix" or not hasattr(os, 'uname'):
# XXX what about the architecture? NT is Intel or Alpha,
# Mac OS is M68k or PPC, etc.
return sys.platform
if sys.version_info < (3, 9):
if os.name == "posix" and hasattr(os, 'uname'):
osname, host, release, version, machine = os.uname()
if osname[:3] == "aix":
from .py38compat import aix_platform
return aix_platform(osname, version, release)
# Try to distinguish various flavours of Unix
return sysconfig.get_platform()
(osname, host, release, version, machine) = os.uname()
# Convert the OS name to lowercase, remove '/' characters, and translate
# spaces (for "Power Macintosh")
osname = osname.lower().replace('/', '')
machine = machine.replace(' ', '_')
machine = machine.replace('/', '-')
if osname[:5] == "linux":
# At least on Linux/Intel, 'machine' is the processor --
# i386, etc.
# XXX what about Alpha, SPARC, etc?
return "%s-%s" % (osname, machine)
elif osname[:5] == "sunos":
if release[0] >= "5": # SunOS 5 == Solaris 2
osname = "solaris"
release = "%d.%s" % (int(release[0]) - 3, release[2:])
# We can't use "platform.architecture()[0]" because a
# bootstrap problem. We use a dict to get an error
# if some suspicious happens.
bitness = {2147483647:"32bit", 9223372036854775807:"64bit"}
machine += ".%s" % bitness[sys.maxsize]
# fall through to standard osname-release-machine representation
elif osname[:3] == "aix":
from .py38compat import aix_platform
return aix_platform(osname, version, release)
elif osname[:6] == "cygwin":
osname = "cygwin"
rel_re = re.compile (r'[\d.]+', re.ASCII)
m = rel_re.match(release)
if m:
release = m.group()
elif osname[:6] == "darwin":
import _osx_support, distutils.sysconfig
osname, release, machine = _osx_support.get_platform_osx(
distutils.sysconfig.get_config_vars(),
osname, release, machine)
return "%s-%s-%s" % (osname, release, machine)
def get_platform():
if os.name == 'nt':
@@ -50,14 +50,14 @@ class Version:
"""
def __init__ (self, vstring=None):
if vstring:
self.parse(vstring)
warnings.warn(
"distutils Version classes are deprecated. "
"Use packaging.version instead.",
DeprecationWarning,
stacklevel=2,
)
if vstring:
self.parse(vstring)
def __repr__ (self):
return "%s ('%s')" % (self.__class__.__name__, str(self))
@@ -1,86 +0,0 @@
import functools
import operator
import itertools
from .extern.jaraco.text import yield_lines
from .extern.jaraco.functools import pass_none
from ._importlib import metadata
from ._itertools import ensure_unique
from .extern.more_itertools import consume
def ensure_valid(ep):
"""
Exercise one of the dynamic properties to trigger
the pattern match.
"""
ep.extras
def load_group(value, group):
"""
Given a value of an entry point or series of entry points,
return each as an EntryPoint.
"""
# normalize to a single sequence of lines
lines = yield_lines(value)
text = f'[{group}]\n' + '\n'.join(lines)
return metadata.EntryPoints._from_text(text)
def by_group_and_name(ep):
return ep.group, ep.name
def validate(eps: metadata.EntryPoints):
"""
Ensure entry points are unique by group and name and validate each.
"""
consume(map(ensure_valid, ensure_unique(eps, key=by_group_and_name)))
return eps
@functools.singledispatch
def load(eps):
"""
Given a Distribution.entry_points, produce EntryPoints.
"""
groups = itertools.chain.from_iterable(
load_group(value, group)
for group, value in eps.items())
return validate(metadata.EntryPoints(groups))
@load.register(str)
def _(eps):
r"""
>>> ep, = load('[console_scripts]\nfoo=bar')
>>> ep.group
'console_scripts'
>>> ep.name
'foo'
>>> ep.value
'bar'
"""
return validate(metadata.EntryPoints(metadata.EntryPoints._from_text(eps)))
load.register(type(None), lambda x: x)
@pass_none
def render(eps: metadata.EntryPoints):
by_group = operator.attrgetter('group')
groups = itertools.groupby(sorted(eps, key=by_group), by_group)
return '\n'.join(
f'[{group}]\n{render_items(items)}\n'
for group, items in groups
)
def render_items(eps):
return '\n'.join(
f'{ep.name} = {ep.value}'
for ep in sorted(eps)
)
@@ -1,36 +0,0 @@
import sys
def disable_importlib_metadata_finder(metadata):
"""
Ensure importlib_metadata doesn't provide older, incompatible
Distributions.
Workaround for #3102.
"""
try:
import importlib_metadata
except ImportError:
return
if importlib_metadata is metadata:
return
to_remove = [
ob
for ob in sys.meta_path
if isinstance(ob, importlib_metadata.MetadataPathFinder)
]
for item in to_remove:
sys.meta_path.remove(item)
if sys.version_info < (3, 10):
from setuptools.extern import importlib_metadata as metadata
disable_importlib_metadata_finder(metadata)
else:
import importlib.metadata as metadata # noqa: F401
if sys.version_info < (3, 9):
from setuptools.extern import importlib_resources as resources
else:
import importlib.resources as resources # noqa: F401
@@ -1,23 +0,0 @@
from setuptools.extern.more_itertools import consume # noqa: F401
# copied from jaraco.itertools 6.1
def ensure_unique(iterable, key=lambda x: x):
"""
Wrap an iterable to raise a ValueError if non-unique values are encountered.
>>> list(ensure_unique('abc'))
['a', 'b', 'c']
>>> consume(ensure_unique('abca'))
Traceback (most recent call last):
...
ValueError: Duplicate element 'a' encountered.
"""
seen = set()
seen_add = seen.add
for element in iterable:
k = key(element)
if k in seen:
raise ValueError(f"Duplicate element {element!r} encountered.")
seen_add(k)
yield element
@@ -1,7 +0,0 @@
import os
def ensure_directory(path):
"""Ensure that the parent directory of `path` exists"""
dirname = os.path.dirname(path)
os.makedirs(dirname, exist_ok=True)
@@ -1,19 +0,0 @@
import setuptools.extern.jaraco.text as text
from pkg_resources import Requirement
def parse_strings(strs):
"""
Yield requirement strings for each specification in `strs`.
`strs` must be a string, or a (possibly-nested) iterable thereof.
"""
return text.join_continuation(map(text.drop_comment, text.yield_lines(strs)))
def parse(strs):
"""
Deprecated drop-in replacement for pkg_resources.parse_requirements.
"""
return map(Requirement, parse_strings(strs))
File diff suppressed because it is too large Load Diff
@@ -1,68 +0,0 @@
import re
import textwrap
import email.message
from ._text import FoldedCase
class Message(email.message.Message):
multiple_use_keys = set(
map(
FoldedCase,
[
'Classifier',
'Obsoletes-Dist',
'Platform',
'Project-URL',
'Provides-Dist',
'Provides-Extra',
'Requires-Dist',
'Requires-External',
'Supported-Platform',
'Dynamic',
],
)
)
"""
Keys that may be indicated multiple times per PEP 566.
"""
def __new__(cls, orig: email.message.Message):
res = super().__new__(cls)
vars(res).update(vars(orig))
return res
def __init__(self, *args, **kwargs):
self._headers = self._repair_headers()
# suppress spurious error from mypy
def __iter__(self):
return super().__iter__()
def _repair_headers(self):
def redent(value):
"Correct for RFC822 indentation"
if not value or '\n' not in value:
return value
return textwrap.dedent(' ' * 8 + value)
headers = [(key, redent(value)) for key, value in vars(self)['_headers']]
if self._payload:
headers.append(('Description', self.get_payload()))
return headers
@property
def json(self):
"""
Convert PackageMetadata to a JSON-compatible format
per PEP 0566.
"""
def transform(key):
value = self.get_all(key) if key in self.multiple_use_keys else self[key]
if key == 'Keywords':
value = re.split(r'\s+', value)
tk = key.lower().replace('-', '_')
return tk, value
return dict(map(transform, map(FoldedCase, self)))
@@ -1,30 +0,0 @@
import collections
# from jaraco.collections 3.3
class FreezableDefaultDict(collections.defaultdict):
"""
Often it is desirable to prevent the mutation of
a default dict after its initial construction, such
as to prevent mutation during iteration.
>>> dd = FreezableDefaultDict(list)
>>> dd[0].append('1')
>>> dd.freeze()
>>> dd[1]
[]
>>> len(dd)
1
"""
def __missing__(self, key):
return getattr(self, '_frozen', super().__missing__)(key)
def freeze(self):
self._frozen = lambda key: self.default_factory()
class Pair(collections.namedtuple('Pair', 'name value')):
@classmethod
def parse(cls, text):
return cls(*map(str.strip, text.split("=", 1)))
@@ -1,71 +0,0 @@
import sys
import platform
__all__ = ['install', 'NullFinder', 'Protocol']
try:
from typing import Protocol
except ImportError: # pragma: no cover
from ..typing_extensions import Protocol # type: ignore
def install(cls):
"""
Class decorator for installation on sys.meta_path.
Adds the backport DistributionFinder to sys.meta_path and
attempts to disable the finder functionality of the stdlib
DistributionFinder.
"""
sys.meta_path.append(cls())
disable_stdlib_finder()
return cls
def disable_stdlib_finder():
"""
Give the backport primacy for discovering path-based distributions
by monkey-patching the stdlib O_O.
See #91 for more background for rationale on this sketchy
behavior.
"""
def matches(finder):
return getattr(
finder, '__module__', None
) == '_frozen_importlib_external' and hasattr(finder, 'find_distributions')
for finder in filter(matches, sys.meta_path): # pragma: nocover
del finder.find_distributions
class NullFinder:
"""
A "Finder" (aka "MetaClassFinder") that never finds any modules,
but may find distributions.
"""
@staticmethod
def find_spec(*args, **kwargs):
return None
# In Python 2, the import system requires finders
# to have a find_module() method, but this usage
# is deprecated in Python 3 in favor of find_spec().
# For the purposes of this finder (i.e. being present
# on sys.meta_path but having no other import
# system functionality), the two methods are identical.
find_module = find_spec
def pypy_partial(val):
"""
Adjust for variable stacklevel on partial under PyPy.
Workaround for #327.
"""
is_pypy = platform.python_implementation() == 'PyPy'
return val + is_pypy
@@ -1,104 +0,0 @@
import types
import functools
# from jaraco.functools 3.3
def method_cache(method, cache_wrapper=None):
"""
Wrap lru_cache to support storing the cache data in the object instances.
Abstracts the common paradigm where the method explicitly saves an
underscore-prefixed protected property on first call and returns that
subsequently.
>>> class MyClass:
... calls = 0
...
... @method_cache
... def method(self, value):
... self.calls += 1
... return value
>>> a = MyClass()
>>> a.method(3)
3
>>> for x in range(75):
... res = a.method(x)
>>> a.calls
75
Note that the apparent behavior will be exactly like that of lru_cache
except that the cache is stored on each instance, so values in one
instance will not flush values from another, and when an instance is
deleted, so are the cached values for that instance.
>>> b = MyClass()
>>> for x in range(35):
... res = b.method(x)
>>> b.calls
35
>>> a.method(0)
0
>>> a.calls
75
Note that if method had been decorated with ``functools.lru_cache()``,
a.calls would have been 76 (due to the cached value of 0 having been
flushed by the 'b' instance).
Clear the cache with ``.cache_clear()``
>>> a.method.cache_clear()
Same for a method that hasn't yet been called.
>>> c = MyClass()
>>> c.method.cache_clear()
Another cache wrapper may be supplied:
>>> cache = functools.lru_cache(maxsize=2)
>>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache)
>>> a = MyClass()
>>> a.method2()
3
Caution - do not subsequently wrap the method with another decorator, such
as ``@property``, which changes the semantics of the function.
See also
http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/
for another implementation and additional justification.
"""
cache_wrapper = cache_wrapper or functools.lru_cache()
def wrapper(self, *args, **kwargs):
# it's the first call, replace the method with a cached, bound method
bound_method = types.MethodType(method, self)
cached_method = cache_wrapper(bound_method)
setattr(self, method.__name__, cached_method)
return cached_method(*args, **kwargs)
# Support cache clear even before cache has been created.
wrapper.cache_clear = lambda: None
return wrapper
# From jaraco.functools 3.3
def pass_none(func):
"""
Wrap func so it's not called if its first param is None
>>> print_text = pass_none(print)
>>> print_text('text')
text
>>> print_text(None)
"""
@functools.wraps(func)
def wrapper(param, *args, **kwargs):
if param is not None:
return func(param, *args, **kwargs)
return wrapper
@@ -1,73 +0,0 @@
from itertools import filterfalse
def unique_everseen(iterable, key=None):
"List unique elements, preserving order. Remember all elements ever seen."
# unique_everseen('AAAABBBCCDAABBB') --> A B C D
# unique_everseen('ABBCcAD', str.lower) --> A B C D
seen = set()
seen_add = seen.add
if key is None:
for element in filterfalse(seen.__contains__, iterable):
seen_add(element)
yield element
else:
for element in iterable:
k = key(element)
if k not in seen:
seen_add(k)
yield element
# copied from more_itertools 8.8
def always_iterable(obj, base_type=(str, bytes)):
"""If *obj* is iterable, return an iterator over its items::
>>> obj = (1, 2, 3)
>>> list(always_iterable(obj))
[1, 2, 3]
If *obj* is not iterable, return a one-item iterable containing *obj*::
>>> obj = 1
>>> list(always_iterable(obj))
[1]
If *obj* is ``None``, return an empty iterable:
>>> obj = None
>>> list(always_iterable(None))
[]
By default, binary and text strings are not considered iterable::
>>> obj = 'foo'
>>> list(always_iterable(obj))
['foo']
If *base_type* is set, objects for which ``isinstance(obj, base_type)``
returns ``True`` won't be considered iterable.
>>> obj = {'a': 1}
>>> list(always_iterable(obj)) # Iterate over the dict's keys
['a']
>>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit
[{'a': 1}]
Set *base_type* to ``None`` to avoid any special handling and treat objects
Python considers iterable as iterable:
>>> obj = 'foo'
>>> list(always_iterable(obj, base_type=None))
['f', 'o', 'o']
"""
if obj is None:
return iter(())
if (base_type is not None) and isinstance(obj, base_type):
return iter((obj,))
try:
return iter(obj)
except TypeError:
return iter((obj,))
@@ -1,48 +0,0 @@
from ._compat import Protocol
from typing import Any, Dict, Iterator, List, TypeVar, Union
_T = TypeVar("_T")
class PackageMetadata(Protocol):
def __len__(self) -> int:
... # pragma: no cover
def __contains__(self, item: str) -> bool:
... # pragma: no cover
def __getitem__(self, key: str) -> str:
... # pragma: no cover
def __iter__(self) -> Iterator[str]:
... # pragma: no cover
def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]:
"""
Return all values associated with a possibly multi-valued key.
"""
@property
def json(self) -> Dict[str, Union[str, List[str]]]:
"""
A JSON-compatible form of the metadata.
"""
class SimplePath(Protocol):
"""
A minimal subset of pathlib.Path required by PathDistribution.
"""
def joinpath(self) -> 'SimplePath':
... # pragma: no cover
def __truediv__(self) -> 'SimplePath':
... # pragma: no cover
def parent(self) -> 'SimplePath':
... # pragma: no cover
def read_text(self) -> str:
... # pragma: no cover
@@ -1,99 +0,0 @@
import re
from ._functools import method_cache
# from jaraco.text 3.5
class FoldedCase(str):
"""
A case insensitive string class; behaves just like str
except compares equal when the only variation is case.
>>> s = FoldedCase('hello world')
>>> s == 'Hello World'
True
>>> 'Hello World' == s
True
>>> s != 'Hello World'
False
>>> s.index('O')
4
>>> s.split('O')
['hell', ' w', 'rld']
>>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta']))
['alpha', 'Beta', 'GAMMA']
Sequence membership is straightforward.
>>> "Hello World" in [s]
True
>>> s in ["Hello World"]
True
You may test for set inclusion, but candidate and elements
must both be folded.
>>> FoldedCase("Hello World") in {s}
True
>>> s in {FoldedCase("Hello World")}
True
String inclusion works as long as the FoldedCase object
is on the right.
>>> "hello" in FoldedCase("Hello World")
True
But not if the FoldedCase object is on the left:
>>> FoldedCase('hello') in 'Hello World'
False
In that case, use in_:
>>> FoldedCase('hello').in_('Hello World')
True
>>> FoldedCase('hello') > FoldedCase('Hello')
False
"""
def __lt__(self, other):
return self.lower() < other.lower()
def __gt__(self, other):
return self.lower() > other.lower()
def __eq__(self, other):
return self.lower() == other.lower()
def __ne__(self, other):
return self.lower() != other.lower()
def __hash__(self):
return hash(self.lower())
def __contains__(self, other):
return super().lower().__contains__(other.lower())
def in_(self, other):
"Does self appear in other?"
return self in FoldedCase(other)
# cache lower since it's likely to be called frequently.
@method_cache
def lower(self):
return super().lower()
def index(self, sub):
return self.lower().index(sub.lower())
def split(self, splitter=' ', maxsplit=0):
pattern = re.compile(re.escape(splitter), re.I)
return pattern.split(self, maxsplit)
@@ -1,36 +0,0 @@
"""Read resources contained within a package."""
from ._common import (
as_file,
files,
Package,
)
from ._legacy import (
contents,
open_binary,
read_binary,
open_text,
read_text,
is_resource,
path,
Resource,
)
from .abc import ResourceReader
__all__ = [
'Package',
'Resource',
'ResourceReader',
'as_file',
'contents',
'files',
'is_resource',
'open_binary',
'open_text',
'path',
'read_binary',
'read_text',
]
@@ -1,170 +0,0 @@
from contextlib import suppress
from io import TextIOWrapper
from . import abc
class SpecLoaderAdapter:
"""
Adapt a package spec to adapt the underlying loader.
"""
def __init__(self, spec, adapter=lambda spec: spec.loader):
self.spec = spec
self.loader = adapter(spec)
def __getattr__(self, name):
return getattr(self.spec, name)
class TraversableResourcesLoader:
"""
Adapt a loader to provide TraversableResources.
"""
def __init__(self, spec):
self.spec = spec
def get_resource_reader(self, name):
return CompatibilityFiles(self.spec)._native()
def _io_wrapper(file, mode='r', *args, **kwargs):
if mode == 'r':
return TextIOWrapper(file, *args, **kwargs)
elif mode == 'rb':
return file
raise ValueError(
"Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode)
)
class CompatibilityFiles:
"""
Adapter for an existing or non-existent resource reader
to provide a compatibility .files().
"""
class SpecPath(abc.Traversable):
"""
Path tied to a module spec.
Can be read and exposes the resource reader children.
"""
def __init__(self, spec, reader):
self._spec = spec
self._reader = reader
def iterdir(self):
if not self._reader:
return iter(())
return iter(
CompatibilityFiles.ChildPath(self._reader, path)
for path in self._reader.contents()
)
def is_file(self):
return False
is_dir = is_file
def joinpath(self, other):
if not self._reader:
return CompatibilityFiles.OrphanPath(other)
return CompatibilityFiles.ChildPath(self._reader, other)
@property
def name(self):
return self._spec.name
def open(self, mode='r', *args, **kwargs):
return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs)
class ChildPath(abc.Traversable):
"""
Path tied to a resource reader child.
Can be read but doesn't expose any meaningful children.
"""
def __init__(self, reader, name):
self._reader = reader
self._name = name
def iterdir(self):
return iter(())
def is_file(self):
return self._reader.is_resource(self.name)
def is_dir(self):
return not self.is_file()
def joinpath(self, other):
return CompatibilityFiles.OrphanPath(self.name, other)
@property
def name(self):
return self._name
def open(self, mode='r', *args, **kwargs):
return _io_wrapper(
self._reader.open_resource(self.name), mode, *args, **kwargs
)
class OrphanPath(abc.Traversable):
"""
Orphan path, not tied to a module spec or resource reader.
Can't be read and doesn't expose any meaningful children.
"""
def __init__(self, *path_parts):
if len(path_parts) < 1:
raise ValueError('Need at least one path part to construct a path')
self._path = path_parts
def iterdir(self):
return iter(())
def is_file(self):
return False
is_dir = is_file
def joinpath(self, other):
return CompatibilityFiles.OrphanPath(*self._path, other)
@property
def name(self):
return self._path[-1]
def open(self, mode='r', *args, **kwargs):
raise FileNotFoundError("Can't open orphan path")
def __init__(self, spec):
self.spec = spec
@property
def _reader(self):
with suppress(AttributeError):
return self.spec.loader.get_resource_reader(self.spec.name)
def _native(self):
"""
Return the native reader if it supports files().
"""
reader = self._reader
return reader if hasattr(reader, 'files') else self
def __getattr__(self, attr):
return getattr(self._reader, attr)
def files(self):
return CompatibilityFiles.SpecPath(self.spec, self._reader)
def wrap_spec(package):
"""
Construct a package spec with traversable compatibility
on the spec/loader/reader.
"""
return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)
@@ -1,104 +0,0 @@
import os
import pathlib
import tempfile
import functools
import contextlib
import types
import importlib
from typing import Union, Optional
from .abc import ResourceReader, Traversable
from ._compat import wrap_spec
Package = Union[types.ModuleType, str]
def files(package):
# type: (Package) -> Traversable
"""
Get a Traversable resource from a package
"""
return from_package(get_package(package))
def get_resource_reader(package):
# type: (types.ModuleType) -> Optional[ResourceReader]
"""
Return the package's loader if it's a ResourceReader.
"""
# We can't use
# a issubclass() check here because apparently abc.'s __subclasscheck__()
# hook wants to create a weak reference to the object, but
# zipimport.zipimporter does not support weak references, resulting in a
# TypeError. That seems terrible.
spec = package.__spec__
reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore
if reader is None:
return None
return reader(spec.name) # type: ignore
def resolve(cand):
# type: (Package) -> types.ModuleType
return cand if isinstance(cand, types.ModuleType) else importlib.import_module(cand)
def get_package(package):
# type: (Package) -> types.ModuleType
"""Take a package name or module object and return the module.
Raise an exception if the resolved module is not a package.
"""
resolved = resolve(package)
if wrap_spec(resolved).submodule_search_locations is None:
raise TypeError(f'{package!r} is not a package')
return resolved
def from_package(package):
"""
Return a Traversable object for the given package.
"""
spec = wrap_spec(package)
reader = spec.loader.get_resource_reader(spec.name)
return reader.files()
@contextlib.contextmanager
def _tempfile(reader, suffix=''):
# Not using tempfile.NamedTemporaryFile as it leads to deeper 'try'
# blocks due to the need to close the temporary file to work on Windows
# properly.
fd, raw_path = tempfile.mkstemp(suffix=suffix)
try:
try:
os.write(fd, reader())
finally:
os.close(fd)
del reader
yield pathlib.Path(raw_path)
finally:
try:
os.remove(raw_path)
except FileNotFoundError:
pass
@functools.singledispatch
def as_file(path):
"""
Given a Traversable object, return that object as a
path on the local file system in a context manager.
"""
return _tempfile(path.read_bytes, suffix=path.name)
@as_file.register(pathlib.Path)
@contextlib.contextmanager
def _(path):
"""
Degenerate behavior for pathlib.Path objects.
"""
yield path
@@ -1,98 +0,0 @@
# flake8: noqa
import abc
import sys
import pathlib
from contextlib import suppress
if sys.version_info >= (3, 10):
from zipfile import Path as ZipPath # type: ignore
else:
from ..zipp import Path as ZipPath # type: ignore
try:
from typing import runtime_checkable # type: ignore
except ImportError:
def runtime_checkable(cls): # type: ignore
return cls
try:
from typing import Protocol # type: ignore
except ImportError:
Protocol = abc.ABC # type: ignore
class TraversableResourcesLoader:
"""
Adapt loaders to provide TraversableResources and other
compatibility.
Used primarily for Python 3.9 and earlier where the native
loaders do not yet implement TraversableResources.
"""
def __init__(self, spec):
self.spec = spec
@property
def path(self):
return self.spec.origin
def get_resource_reader(self, name):
from . import readers, _adapters
def _zip_reader(spec):
with suppress(AttributeError):
return readers.ZipReader(spec.loader, spec.name)
def _namespace_reader(spec):
with suppress(AttributeError, ValueError):
return readers.NamespaceReader(spec.submodule_search_locations)
def _available_reader(spec):
with suppress(AttributeError):
return spec.loader.get_resource_reader(spec.name)
def _native_reader(spec):
reader = _available_reader(spec)
return reader if hasattr(reader, 'files') else None
def _file_reader(spec):
try:
path = pathlib.Path(self.path)
except TypeError:
return None
if path.exists():
return readers.FileReader(self)
return (
# native reader if it supplies 'files'
_native_reader(self.spec)
or
# local ZipReader if a zip module
_zip_reader(self.spec)
or
# local NamespaceReader if a namespace module
_namespace_reader(self.spec)
or
# local FileReader
_file_reader(self.spec)
# fallback - adapt the spec ResourceReader to TraversableReader
or _adapters.CompatibilityFiles(self.spec)
)
def wrap_spec(package):
"""
Construct a package spec with traversable compatibility
on the spec/loader/reader.
Supersedes _adapters.wrap_spec to use TraversableResourcesLoader
from above for older Python compatibility (<3.10).
"""
from . import _adapters
return _adapters.SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)
@@ -1,35 +0,0 @@
from itertools import filterfalse
from typing import (
Callable,
Iterable,
Iterator,
Optional,
Set,
TypeVar,
Union,
)
# Type and type variable definitions
_T = TypeVar('_T')
_U = TypeVar('_U')
def unique_everseen(
iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None
) -> Iterator[_T]:
"List unique elements, preserving order. Remember all elements ever seen."
# unique_everseen('AAAABBBCCDAABBB') --> A B C D
# unique_everseen('ABBCcAD', str.lower) --> A B C D
seen: Set[Union[_T, _U]] = set()
seen_add = seen.add
if key is None:
for element in filterfalse(seen.__contains__, iterable):
seen_add(element)
yield element
else:
for element in iterable:
k = key(element)
if k not in seen:
seen_add(k)
yield element
@@ -1,121 +0,0 @@
import functools
import os
import pathlib
import types
import warnings
from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any
from . import _common
Package = Union[types.ModuleType, str]
Resource = str
def deprecated(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
warnings.warn(
f"{func.__name__} is deprecated. Use files() instead. "
"Refer to https://importlib-resources.readthedocs.io"
"/en/latest/using.html#migrating-from-legacy for migration advice.",
DeprecationWarning,
stacklevel=2,
)
return func(*args, **kwargs)
return wrapper
def normalize_path(path):
# type: (Any) -> str
"""Normalize a path by ensuring it is a string.
If the resulting string contains path separators, an exception is raised.
"""
str_path = str(path)
parent, file_name = os.path.split(str_path)
if parent:
raise ValueError(f'{path!r} must be only a file name')
return file_name
@deprecated
def open_binary(package: Package, resource: Resource) -> BinaryIO:
"""Return a file-like object opened for binary reading of the resource."""
return (_common.files(package) / normalize_path(resource)).open('rb')
@deprecated
def read_binary(package: Package, resource: Resource) -> bytes:
"""Return the binary contents of the resource."""
return (_common.files(package) / normalize_path(resource)).read_bytes()
@deprecated
def open_text(
package: Package,
resource: Resource,
encoding: str = 'utf-8',
errors: str = 'strict',
) -> TextIO:
"""Return a file-like object opened for text reading of the resource."""
return (_common.files(package) / normalize_path(resource)).open(
'r', encoding=encoding, errors=errors
)
@deprecated
def read_text(
package: Package,
resource: Resource,
encoding: str = 'utf-8',
errors: str = 'strict',
) -> str:
"""Return the decoded string of the resource.
The decoding-related arguments have the same semantics as those of
bytes.decode().
"""
with open_text(package, resource, encoding, errors) as fp:
return fp.read()
@deprecated
def contents(package: Package) -> Iterable[str]:
"""Return an iterable of entries in `package`.
Note that not all entries are resources. Specifically, directories are
not considered resources. Use `is_resource()` on each entry returned here
to check if it is a resource or not.
"""
return [path.name for path in _common.files(package).iterdir()]
@deprecated
def is_resource(package: Package, name: str) -> bool:
"""True if `name` is a resource inside `package`.
Directories are *not* resources.
"""
resource = normalize_path(name)
return any(
traversable.name == resource and traversable.is_file()
for traversable in _common.files(package).iterdir()
)
@deprecated
def path(
package: Package,
resource: Resource,
) -> ContextManager[pathlib.Path]:
"""A context manager providing a file path object to the resource.
If the resource does not already exist on its own on the file system,
a temporary file will be created. If the file was created, the file
will be deleted upon exiting the context manager (no exception is
raised if the file was deleted prior to the context manager
exiting).
"""
return _common.as_file(_common.files(package) / normalize_path(resource))
@@ -1,137 +0,0 @@
import abc
from typing import BinaryIO, Iterable, Text
from ._compat import runtime_checkable, Protocol
class ResourceReader(metaclass=abc.ABCMeta):
"""Abstract base class for loaders to provide resource reading support."""
@abc.abstractmethod
def open_resource(self, resource: Text) -> BinaryIO:
"""Return an opened, file-like object for binary reading.
The 'resource' argument is expected to represent only a file name.
If the resource cannot be found, FileNotFoundError is raised.
"""
# This deliberately raises FileNotFoundError instead of
# NotImplementedError so that if this method is accidentally called,
# it'll still do the right thing.
raise FileNotFoundError
@abc.abstractmethod
def resource_path(self, resource: Text) -> Text:
"""Return the file system path to the specified resource.
The 'resource' argument is expected to represent only a file name.
If the resource does not exist on the file system, raise
FileNotFoundError.
"""
# This deliberately raises FileNotFoundError instead of
# NotImplementedError so that if this method is accidentally called,
# it'll still do the right thing.
raise FileNotFoundError
@abc.abstractmethod
def is_resource(self, path: Text) -> bool:
"""Return True if the named 'path' is a resource.
Files are resources, directories are not.
"""
raise FileNotFoundError
@abc.abstractmethod
def contents(self) -> Iterable[str]:
"""Return an iterable of entries in `package`."""
raise FileNotFoundError
@runtime_checkable
class Traversable(Protocol):
"""
An object with a subset of pathlib.Path methods suitable for
traversing directories and opening files.
"""
@abc.abstractmethod
def iterdir(self):
"""
Yield Traversable objects in self
"""
def read_bytes(self):
"""
Read contents of self as bytes
"""
with self.open('rb') as strm:
return strm.read()
def read_text(self, encoding=None):
"""
Read contents of self as text
"""
with self.open(encoding=encoding) as strm:
return strm.read()
@abc.abstractmethod
def is_dir(self) -> bool:
"""
Return True if self is a directory
"""
@abc.abstractmethod
def is_file(self) -> bool:
"""
Return True if self is a file
"""
@abc.abstractmethod
def joinpath(self, child):
"""
Return Traversable child in self
"""
def __truediv__(self, child):
"""
Return Traversable child in self
"""
return self.joinpath(child)
@abc.abstractmethod
def open(self, mode='r', *args, **kwargs):
"""
mode may be 'r' or 'rb' to open as text or binary. Return a handle
suitable for reading (same as pathlib.Path.open).
When opening as text, accepts encoding parameters such as those
accepted by io.TextIOWrapper.
"""
@abc.abstractproperty
def name(self) -> str:
"""
The base name of this object without any parent references.
"""
class TraversableResources(ResourceReader):
"""
The required interface for providing traversable
resources.
"""
@abc.abstractmethod
def files(self):
"""Return a Traversable object for the loaded package."""
def open_resource(self, resource):
return self.files().joinpath(resource).open('rb')
def resource_path(self, resource):
raise FileNotFoundError(resource)
def is_resource(self, path):
return self.files().joinpath(path).is_file()
def contents(self):
return (item.name for item in self.files().iterdir())
@@ -1,122 +0,0 @@
import collections
import pathlib
import operator
from . import abc
from ._itertools import unique_everseen
from ._compat import ZipPath
def remove_duplicates(items):
return iter(collections.OrderedDict.fromkeys(items))
class FileReader(abc.TraversableResources):
def __init__(self, loader):
self.path = pathlib.Path(loader.path).parent
def resource_path(self, resource):
"""
Return the file system path to prevent
`resources.path()` from creating a temporary
copy.
"""
return str(self.path.joinpath(resource))
def files(self):
return self.path
class ZipReader(abc.TraversableResources):
def __init__(self, loader, module):
_, _, name = module.rpartition('.')
self.prefix = loader.prefix.replace('\\', '/') + name + '/'
self.archive = loader.archive
def open_resource(self, resource):
try:
return super().open_resource(resource)
except KeyError as exc:
raise FileNotFoundError(exc.args[0])
def is_resource(self, path):
# workaround for `zipfile.Path.is_file` returning true
# for non-existent paths.
target = self.files().joinpath(path)
return target.is_file() and target.exists()
def files(self):
return ZipPath(self.archive, self.prefix)
class MultiplexedPath(abc.Traversable):
"""
Given a series of Traversable objects, implement a merged
version of the interface across all objects. Useful for
namespace packages which may be multihomed at a single
name.
"""
def __init__(self, *paths):
self._paths = list(map(pathlib.Path, remove_duplicates(paths)))
if not self._paths:
message = 'MultiplexedPath must contain at least one path'
raise FileNotFoundError(message)
if not all(path.is_dir() for path in self._paths):
raise NotADirectoryError('MultiplexedPath only supports directories')
def iterdir(self):
files = (file for path in self._paths for file in path.iterdir())
return unique_everseen(files, key=operator.attrgetter('name'))
def read_bytes(self):
raise FileNotFoundError(f'{self} is not a file')
def read_text(self, *args, **kwargs):
raise FileNotFoundError(f'{self} is not a file')
def is_dir(self):
return True
def is_file(self):
return False
def joinpath(self, child):
# first try to find child in current paths
for file in self.iterdir():
if file.name == child:
return file
# if it does not exist, construct it with the first path
return self._paths[0] / child
__truediv__ = joinpath
def open(self, *args, **kwargs):
raise FileNotFoundError(f'{self} is not a file')
@property
def name(self):
return self._paths[0].name
def __repr__(self):
paths = ', '.join(f"'{path}'" for path in self._paths)
return f'MultiplexedPath({paths})'
class NamespaceReader(abc.TraversableResources):
def __init__(self, namespace_path):
if 'NamespacePath' not in str(namespace_path):
raise ValueError('Invalid path')
self.path = MultiplexedPath(*list(namespace_path))
def resource_path(self, resource):
"""
Return the file system path to prevent
`resources.path()` from creating a temporary
copy.
"""
return str(self.path.joinpath(resource))
def files(self):
return self.path
@@ -1,116 +0,0 @@
"""
Interface adapters for low-level readers.
"""
import abc
import io
import itertools
from typing import BinaryIO, List
from .abc import Traversable, TraversableResources
class SimpleReader(abc.ABC):
"""
The minimum, low-level interface required from a resource
provider.
"""
@abc.abstractproperty
def package(self):
# type: () -> str
"""
The name of the package for which this reader loads resources.
"""
@abc.abstractmethod
def children(self):
# type: () -> List['SimpleReader']
"""
Obtain an iterable of SimpleReader for available
child containers (e.g. directories).
"""
@abc.abstractmethod
def resources(self):
# type: () -> List[str]
"""
Obtain available named resources for this virtual package.
"""
@abc.abstractmethod
def open_binary(self, resource):
# type: (str) -> BinaryIO
"""
Obtain a File-like for a named resource.
"""
@property
def name(self):
return self.package.split('.')[-1]
class ResourceHandle(Traversable):
"""
Handle to a named resource in a ResourceReader.
"""
def __init__(self, parent, name):
# type: (ResourceContainer, str) -> None
self.parent = parent
self.name = name # type: ignore
def is_file(self):
return True
def is_dir(self):
return False
def open(self, mode='r', *args, **kwargs):
stream = self.parent.reader.open_binary(self.name)
if 'b' not in mode:
stream = io.TextIOWrapper(*args, **kwargs)
return stream
def joinpath(self, name):
raise RuntimeError("Cannot traverse into a resource")
class ResourceContainer(Traversable):
"""
Traversable container for a package's resources via its reader.
"""
def __init__(self, reader):
# type: (SimpleReader) -> None
self.reader = reader
def is_dir(self):
return True
def is_file(self):
return False
def iterdir(self):
files = (ResourceHandle(self, name) for name in self.reader.resources)
dirs = map(ResourceContainer, self.reader.children())
return itertools.chain(files, dirs)
def open(self, *args, **kwargs):
raise IsADirectoryError()
def joinpath(self, name):
return next(
traversable for traversable in self.iterdir() if traversable.name == name
)
class TraversableReader(TraversableResources, SimpleReader):
"""
A TraversableResources based on SimpleReader. Resource providers
may derive from this class to provide the TraversableResources
interface by supplying the SimpleReader interface.
"""
def files(self):
return ResourceContainer(self)
@@ -1,213 +0,0 @@
import os
import subprocess
import contextlib
import functools
import tempfile
import shutil
import operator
@contextlib.contextmanager
def pushd(dir):
orig = os.getcwd()
os.chdir(dir)
try:
yield dir
finally:
os.chdir(orig)
@contextlib.contextmanager
def tarball_context(url, target_dir=None, runner=None, pushd=pushd):
"""
Get a tarball, extract it, change to that directory, yield, then
clean up.
`runner` is the function to invoke commands.
`pushd` is a context manager for changing the directory.
"""
if target_dir is None:
target_dir = os.path.basename(url).replace('.tar.gz', '').replace('.tgz', '')
if runner is None:
runner = functools.partial(subprocess.check_call, shell=True)
# In the tar command, use --strip-components=1 to strip the first path and
# then
# use -C to cause the files to be extracted to {target_dir}. This ensures
# that we always know where the files were extracted.
runner('mkdir {target_dir}'.format(**vars()))
try:
getter = 'wget {url} -O -'
extract = 'tar x{compression} --strip-components=1 -C {target_dir}'
cmd = ' | '.join((getter, extract))
runner(cmd.format(compression=infer_compression(url), **vars()))
with pushd(target_dir):
yield target_dir
finally:
runner('rm -Rf {target_dir}'.format(**vars()))
def infer_compression(url):
"""
Given a URL or filename, infer the compression code for tar.
"""
# cheat and just assume it's the last two characters
compression_indicator = url[-2:]
mapping = dict(gz='z', bz='j', xz='J')
# Assume 'z' (gzip) if no match
return mapping.get(compression_indicator, 'z')
@contextlib.contextmanager
def temp_dir(remover=shutil.rmtree):
"""
Create a temporary directory context. Pass a custom remover
to override the removal behavior.
"""
temp_dir = tempfile.mkdtemp()
try:
yield temp_dir
finally:
remover(temp_dir)
@contextlib.contextmanager
def repo_context(url, branch=None, quiet=True, dest_ctx=temp_dir):
"""
Check out the repo indicated by url.
If dest_ctx is supplied, it should be a context manager
to yield the target directory for the check out.
"""
exe = 'git' if 'git' in url else 'hg'
with dest_ctx() as repo_dir:
cmd = [exe, 'clone', url, repo_dir]
if branch:
cmd.extend(['--branch', branch])
devnull = open(os.path.devnull, 'w')
stdout = devnull if quiet else None
subprocess.check_call(cmd, stdout=stdout)
yield repo_dir
@contextlib.contextmanager
def null():
yield
class ExceptionTrap:
"""
A context manager that will catch certain exceptions and provide an
indication they occurred.
>>> with ExceptionTrap() as trap:
... raise Exception()
>>> bool(trap)
True
>>> with ExceptionTrap() as trap:
... pass
>>> bool(trap)
False
>>> with ExceptionTrap(ValueError) as trap:
... raise ValueError("1 + 1 is not 3")
>>> bool(trap)
True
>>> with ExceptionTrap(ValueError) as trap:
... raise Exception()
Traceback (most recent call last):
...
Exception
>>> bool(trap)
False
"""
exc_info = None, None, None
def __init__(self, exceptions=(Exception,)):
self.exceptions = exceptions
def __enter__(self):
return self
@property
def type(self):
return self.exc_info[0]
@property
def value(self):
return self.exc_info[1]
@property
def tb(self):
return self.exc_info[2]
def __exit__(self, *exc_info):
type = exc_info[0]
matches = type and issubclass(type, self.exceptions)
if matches:
self.exc_info = exc_info
return matches
def __bool__(self):
return bool(self.type)
def raises(self, func, *, _test=bool):
"""
Wrap func and replace the result with the truth
value of the trap (True if an exception occurred).
First, give the decorator an alias to support Python 3.8
Syntax.
>>> raises = ExceptionTrap(ValueError).raises
Now decorate a function that always fails.
>>> @raises
... def fail():
... raise ValueError('failed')
>>> fail()
True
"""
@functools.wraps(func)
def wrapper(*args, **kwargs):
with ExceptionTrap(self.exceptions) as trap:
func(*args, **kwargs)
return _test(trap)
return wrapper
def passes(self, func):
"""
Wrap func and replace the result with the truth
value of the trap (True if no exception).
First, give the decorator an alias to support Python 3.8
Syntax.
>>> passes = ExceptionTrap(ValueError).passes
Now decorate a function that always fails.
>>> @passes
... def fail():
... raise ValueError('failed')
>>> fail()
False
"""
return self.raises(func, _test=operator.not_)
class suppress(contextlib.suppress, contextlib.ContextDecorator):
"""
A version of contextlib.suppress with decorator support.
>>> @suppress(KeyError)
... def key_error():
... {}['']
>>> key_error()
"""
@@ -1,525 +0,0 @@
import functools
import time
import inspect
import collections
import types
import itertools
import setuptools.extern.more_itertools
from typing import Callable, TypeVar
CallableT = TypeVar("CallableT", bound=Callable[..., object])
def compose(*funcs):
"""
Compose any number of unary functions into a single unary function.
>>> import textwrap
>>> expected = str.strip(textwrap.dedent(compose.__doc__))
>>> strip_and_dedent = compose(str.strip, textwrap.dedent)
>>> strip_and_dedent(compose.__doc__) == expected
True
Compose also allows the innermost function to take arbitrary arguments.
>>> round_three = lambda x: round(x, ndigits=3)
>>> f = compose(round_three, int.__truediv__)
>>> [f(3*x, x+1) for x in range(1,10)]
[1.5, 2.0, 2.25, 2.4, 2.5, 2.571, 2.625, 2.667, 2.7]
"""
def compose_two(f1, f2):
return lambda *args, **kwargs: f1(f2(*args, **kwargs))
return functools.reduce(compose_two, funcs)
def method_caller(method_name, *args, **kwargs):
"""
Return a function that will call a named method on the
target object with optional positional and keyword
arguments.
>>> lower = method_caller('lower')
>>> lower('MyString')
'mystring'
"""
def call_method(target):
func = getattr(target, method_name)
return func(*args, **kwargs)
return call_method
def once(func):
"""
Decorate func so it's only ever called the first time.
This decorator can ensure that an expensive or non-idempotent function
will not be expensive on subsequent calls and is idempotent.
>>> add_three = once(lambda a: a+3)
>>> add_three(3)
6
>>> add_three(9)
6
>>> add_three('12')
6
To reset the stored value, simply clear the property ``saved_result``.
>>> del add_three.saved_result
>>> add_three(9)
12
>>> add_three(8)
12
Or invoke 'reset()' on it.
>>> add_three.reset()
>>> add_three(-3)
0
>>> add_three(0)
0
"""
@functools.wraps(func)
def wrapper(*args, **kwargs):
if not hasattr(wrapper, 'saved_result'):
wrapper.saved_result = func(*args, **kwargs)
return wrapper.saved_result
wrapper.reset = lambda: vars(wrapper).__delitem__('saved_result')
return wrapper
def method_cache(
method: CallableT,
cache_wrapper: Callable[
[CallableT], CallableT
] = functools.lru_cache(), # type: ignore[assignment]
) -> CallableT:
"""
Wrap lru_cache to support storing the cache data in the object instances.
Abstracts the common paradigm where the method explicitly saves an
underscore-prefixed protected property on first call and returns that
subsequently.
>>> class MyClass:
... calls = 0
...
... @method_cache
... def method(self, value):
... self.calls += 1
... return value
>>> a = MyClass()
>>> a.method(3)
3
>>> for x in range(75):
... res = a.method(x)
>>> a.calls
75
Note that the apparent behavior will be exactly like that of lru_cache
except that the cache is stored on each instance, so values in one
instance will not flush values from another, and when an instance is
deleted, so are the cached values for that instance.
>>> b = MyClass()
>>> for x in range(35):
... res = b.method(x)
>>> b.calls
35
>>> a.method(0)
0
>>> a.calls
75
Note that if method had been decorated with ``functools.lru_cache()``,
a.calls would have been 76 (due to the cached value of 0 having been
flushed by the 'b' instance).
Clear the cache with ``.cache_clear()``
>>> a.method.cache_clear()
Same for a method that hasn't yet been called.
>>> c = MyClass()
>>> c.method.cache_clear()
Another cache wrapper may be supplied:
>>> cache = functools.lru_cache(maxsize=2)
>>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache)
>>> a = MyClass()
>>> a.method2()
3
Caution - do not subsequently wrap the method with another decorator, such
as ``@property``, which changes the semantics of the function.
See also
http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/
for another implementation and additional justification.
"""
def wrapper(self: object, *args: object, **kwargs: object) -> object:
# it's the first call, replace the method with a cached, bound method
bound_method: CallableT = types.MethodType( # type: ignore[assignment]
method, self
)
cached_method = cache_wrapper(bound_method)
setattr(self, method.__name__, cached_method)
return cached_method(*args, **kwargs)
# Support cache clear even before cache has been created.
wrapper.cache_clear = lambda: None # type: ignore[attr-defined]
return ( # type: ignore[return-value]
_special_method_cache(method, cache_wrapper) or wrapper
)
def _special_method_cache(method, cache_wrapper):
"""
Because Python treats special methods differently, it's not
possible to use instance attributes to implement the cached
methods.
Instead, install the wrapper method under a different name
and return a simple proxy to that wrapper.
https://github.com/jaraco/jaraco.functools/issues/5
"""
name = method.__name__
special_names = '__getattr__', '__getitem__'
if name not in special_names:
return
wrapper_name = '__cached' + name
def proxy(self, *args, **kwargs):
if wrapper_name not in vars(self):
bound = types.MethodType(method, self)
cache = cache_wrapper(bound)
setattr(self, wrapper_name, cache)
else:
cache = getattr(self, wrapper_name)
return cache(*args, **kwargs)
return proxy
def apply(transform):
"""
Decorate a function with a transform function that is
invoked on results returned from the decorated function.
>>> @apply(reversed)
... def get_numbers(start):
... "doc for get_numbers"
... return range(start, start+3)
>>> list(get_numbers(4))
[6, 5, 4]
>>> get_numbers.__doc__
'doc for get_numbers'
"""
def wrap(func):
return functools.wraps(func)(compose(transform, func))
return wrap
def result_invoke(action):
r"""
Decorate a function with an action function that is
invoked on the results returned from the decorated
function (for its side-effect), then return the original
result.
>>> @result_invoke(print)
... def add_two(a, b):
... return a + b
>>> x = add_two(2, 3)
5
>>> x
5
"""
def wrap(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
action(result)
return result
return wrapper
return wrap
def call_aside(f, *args, **kwargs):
"""
Call a function for its side effect after initialization.
>>> @call_aside
... def func(): print("called")
called
>>> func()
called
Use functools.partial to pass parameters to the initial call
>>> @functools.partial(call_aside, name='bingo')
... def func(name): print("called with", name)
called with bingo
"""
f(*args, **kwargs)
return f
class Throttler:
"""
Rate-limit a function (or other callable)
"""
def __init__(self, func, max_rate=float('Inf')):
if isinstance(func, Throttler):
func = func.func
self.func = func
self.max_rate = max_rate
self.reset()
def reset(self):
self.last_called = 0
def __call__(self, *args, **kwargs):
self._wait()
return self.func(*args, **kwargs)
def _wait(self):
"ensure at least 1/max_rate seconds from last call"
elapsed = time.time() - self.last_called
must_wait = 1 / self.max_rate - elapsed
time.sleep(max(0, must_wait))
self.last_called = time.time()
def __get__(self, obj, type=None):
return first_invoke(self._wait, functools.partial(self.func, obj))
def first_invoke(func1, func2):
"""
Return a function that when invoked will invoke func1 without
any parameters (for its side-effect) and then invoke func2
with whatever parameters were passed, returning its result.
"""
def wrapper(*args, **kwargs):
func1()
return func2(*args, **kwargs)
return wrapper
def retry_call(func, cleanup=lambda: None, retries=0, trap=()):
"""
Given a callable func, trap the indicated exceptions
for up to 'retries' times, invoking cleanup on the
exception. On the final attempt, allow any exceptions
to propagate.
"""
attempts = itertools.count() if retries == float('inf') else range(retries)
for attempt in attempts:
try:
return func()
except trap:
cleanup()
return func()
def retry(*r_args, **r_kwargs):
"""
Decorator wrapper for retry_call. Accepts arguments to retry_call
except func and then returns a decorator for the decorated function.
Ex:
>>> @retry(retries=3)
... def my_func(a, b):
... "this is my funk"
... print(a, b)
>>> my_func.__doc__
'this is my funk'
"""
def decorate(func):
@functools.wraps(func)
def wrapper(*f_args, **f_kwargs):
bound = functools.partial(func, *f_args, **f_kwargs)
return retry_call(bound, *r_args, **r_kwargs)
return wrapper
return decorate
def print_yielded(func):
"""
Convert a generator into a function that prints all yielded elements
>>> @print_yielded
... def x():
... yield 3; yield None
>>> x()
3
None
"""
print_all = functools.partial(map, print)
print_results = compose(more_itertools.consume, print_all, func)
return functools.wraps(func)(print_results)
def pass_none(func):
"""
Wrap func so it's not called if its first param is None
>>> print_text = pass_none(print)
>>> print_text('text')
text
>>> print_text(None)
"""
@functools.wraps(func)
def wrapper(param, *args, **kwargs):
if param is not None:
return func(param, *args, **kwargs)
return wrapper
def assign_params(func, namespace):
"""
Assign parameters from namespace where func solicits.
>>> def func(x, y=3):
... print(x, y)
>>> assigned = assign_params(func, dict(x=2, z=4))
>>> assigned()
2 3
The usual errors are raised if a function doesn't receive
its required parameters:
>>> assigned = assign_params(func, dict(y=3, z=4))
>>> assigned()
Traceback (most recent call last):
TypeError: func() ...argument...
It even works on methods:
>>> class Handler:
... def meth(self, arg):
... print(arg)
>>> assign_params(Handler().meth, dict(arg='crystal', foo='clear'))()
crystal
"""
sig = inspect.signature(func)
params = sig.parameters.keys()
call_ns = {k: namespace[k] for k in params if k in namespace}
return functools.partial(func, **call_ns)
def save_method_args(method):
"""
Wrap a method such that when it is called, the args and kwargs are
saved on the method.
>>> class MyClass:
... @save_method_args
... def method(self, a, b):
... print(a, b)
>>> my_ob = MyClass()
>>> my_ob.method(1, 2)
1 2
>>> my_ob._saved_method.args
(1, 2)
>>> my_ob._saved_method.kwargs
{}
>>> my_ob.method(a=3, b='foo')
3 foo
>>> my_ob._saved_method.args
()
>>> my_ob._saved_method.kwargs == dict(a=3, b='foo')
True
The arguments are stored on the instance, allowing for
different instance to save different args.
>>> your_ob = MyClass()
>>> your_ob.method({str('x'): 3}, b=[4])
{'x': 3} [4]
>>> your_ob._saved_method.args
({'x': 3},)
>>> my_ob._saved_method.args
()
"""
args_and_kwargs = collections.namedtuple('args_and_kwargs', 'args kwargs')
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
attr_name = '_saved_' + method.__name__
attr = args_and_kwargs(args, kwargs)
setattr(self, attr_name, attr)
return method(self, *args, **kwargs)
return wrapper
def except_(*exceptions, replace=None, use=None):
"""
Replace the indicated exceptions, if raised, with the indicated
literal replacement or evaluated expression (if present).
>>> safe_int = except_(ValueError)(int)
>>> safe_int('five')
>>> safe_int('5')
5
Specify a literal replacement with ``replace``.
>>> safe_int_r = except_(ValueError, replace=0)(int)
>>> safe_int_r('five')
0
Provide an expression to ``use`` to pass through particular parameters.
>>> safe_int_pt = except_(ValueError, use='args[0]')(int)
>>> safe_int_pt('five')
'five'
"""
def decorate(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except exceptions:
try:
return eval(use)
except TypeError:
return replace
return wrapper
return decorate
@@ -1,599 +0,0 @@
import re
import itertools
import textwrap
import functools
try:
from importlib.resources import files # type: ignore
except ImportError: # pragma: nocover
from setuptools.extern.importlib_resources import files # type: ignore
from setuptools.extern.jaraco.functools import compose, method_cache
from setuptools.extern.jaraco.context import ExceptionTrap
def substitution(old, new):
"""
Return a function that will perform a substitution on a string
"""
return lambda s: s.replace(old, new)
def multi_substitution(*substitutions):
"""
Take a sequence of pairs specifying substitutions, and create
a function that performs those substitutions.
>>> multi_substitution(('foo', 'bar'), ('bar', 'baz'))('foo')
'baz'
"""
substitutions = itertools.starmap(substitution, substitutions)
# compose function applies last function first, so reverse the
# substitutions to get the expected order.
substitutions = reversed(tuple(substitutions))
return compose(*substitutions)
class FoldedCase(str):
"""
A case insensitive string class; behaves just like str
except compares equal when the only variation is case.
>>> s = FoldedCase('hello world')
>>> s == 'Hello World'
True
>>> 'Hello World' == s
True
>>> s != 'Hello World'
False
>>> s.index('O')
4
>>> s.split('O')
['hell', ' w', 'rld']
>>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta']))
['alpha', 'Beta', 'GAMMA']
Sequence membership is straightforward.
>>> "Hello World" in [s]
True
>>> s in ["Hello World"]
True
You may test for set inclusion, but candidate and elements
must both be folded.
>>> FoldedCase("Hello World") in {s}
True
>>> s in {FoldedCase("Hello World")}
True
String inclusion works as long as the FoldedCase object
is on the right.
>>> "hello" in FoldedCase("Hello World")
True
But not if the FoldedCase object is on the left:
>>> FoldedCase('hello') in 'Hello World'
False
In that case, use ``in_``:
>>> FoldedCase('hello').in_('Hello World')
True
>>> FoldedCase('hello') > FoldedCase('Hello')
False
"""
def __lt__(self, other):
return self.lower() < other.lower()
def __gt__(self, other):
return self.lower() > other.lower()
def __eq__(self, other):
return self.lower() == other.lower()
def __ne__(self, other):
return self.lower() != other.lower()
def __hash__(self):
return hash(self.lower())
def __contains__(self, other):
return super().lower().__contains__(other.lower())
def in_(self, other):
"Does self appear in other?"
return self in FoldedCase(other)
# cache lower since it's likely to be called frequently.
@method_cache
def lower(self):
return super().lower()
def index(self, sub):
return self.lower().index(sub.lower())
def split(self, splitter=' ', maxsplit=0):
pattern = re.compile(re.escape(splitter), re.I)
return pattern.split(self, maxsplit)
# Python 3.8 compatibility
_unicode_trap = ExceptionTrap(UnicodeDecodeError)
@_unicode_trap.passes
def is_decodable(value):
r"""
Return True if the supplied value is decodable (using the default
encoding).
>>> is_decodable(b'\xff')
False
>>> is_decodable(b'\x32')
True
"""
value.decode()
def is_binary(value):
r"""
Return True if the value appears to be binary (that is, it's a byte
string and isn't decodable).
>>> is_binary(b'\xff')
True
>>> is_binary('\xff')
False
"""
return isinstance(value, bytes) and not is_decodable(value)
def trim(s):
r"""
Trim something like a docstring to remove the whitespace that
is common due to indentation and formatting.
>>> trim("\n\tfoo = bar\n\t\tbar = baz\n")
'foo = bar\n\tbar = baz'
"""
return textwrap.dedent(s).strip()
def wrap(s):
"""
Wrap lines of text, retaining existing newlines as
paragraph markers.
>>> print(wrap(lorem_ipsum))
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad
minim veniam, quis nostrud exercitation ullamco laboris nisi ut
aliquip ex ea commodo consequat. Duis aute irure dolor in
reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
pariatur. Excepteur sint occaecat cupidatat non proident, sunt in
culpa qui officia deserunt mollit anim id est laborum.
<BLANKLINE>
Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam
varius, turpis et commodo pharetra, est eros bibendum elit, nec luctus
magna felis sollicitudin mauris. Integer in mauris eu nibh euismod
gravida. Duis ac tellus et risus vulputate vehicula. Donec lobortis
risus a elit. Etiam tempor. Ut ullamcorper, ligula eu tempor congue,
eros est euismod turpis, id tincidunt sapien risus a quam. Maecenas
fermentum consequat mi. Donec fermentum. Pellentesque malesuada nulla
a mi. Duis sapien sem, aliquet nec, commodo eget, consequat quis,
neque. Aliquam faucibus, elit ut dictum aliquet, felis nisl adipiscing
sapien, sed malesuada diam lacus eget erat. Cras mollis scelerisque
nunc. Nullam arcu. Aliquam consequat. Curabitur augue lorem, dapibus
quis, laoreet et, pretium ac, nisi. Aenean magna nisl, mollis quis,
molestie eu, feugiat in, orci. In hac habitasse platea dictumst.
"""
paragraphs = s.splitlines()
wrapped = ('\n'.join(textwrap.wrap(para)) for para in paragraphs)
return '\n\n'.join(wrapped)
def unwrap(s):
r"""
Given a multi-line string, return an unwrapped version.
>>> wrapped = wrap(lorem_ipsum)
>>> wrapped.count('\n')
20
>>> unwrapped = unwrap(wrapped)
>>> unwrapped.count('\n')
1
>>> print(unwrapped)
Lorem ipsum dolor sit amet, consectetur adipiscing ...
Curabitur pretium tincidunt lacus. Nulla gravida orci ...
"""
paragraphs = re.split(r'\n\n+', s)
cleaned = (para.replace('\n', ' ') for para in paragraphs)
return '\n'.join(cleaned)
class Splitter(object):
"""object that will split a string with the given arguments for each call
>>> s = Splitter(',')
>>> s('hello, world, this is your, master calling')
['hello', ' world', ' this is your', ' master calling']
"""
def __init__(self, *args):
self.args = args
def __call__(self, s):
return s.split(*self.args)
def indent(string, prefix=' ' * 4):
"""
>>> indent('foo')
' foo'
"""
return prefix + string
class WordSet(tuple):
"""
Given an identifier, return the words that identifier represents,
whether in camel case, underscore-separated, etc.
>>> WordSet.parse("camelCase")
('camel', 'Case')
>>> WordSet.parse("under_sep")
('under', 'sep')
Acronyms should be retained
>>> WordSet.parse("firstSNL")
('first', 'SNL')
>>> WordSet.parse("you_and_I")
('you', 'and', 'I')
>>> WordSet.parse("A simple test")
('A', 'simple', 'test')
Multiple caps should not interfere with the first cap of another word.
>>> WordSet.parse("myABCClass")
('my', 'ABC', 'Class')
The result is a WordSet, so you can get the form you need.
>>> WordSet.parse("myABCClass").underscore_separated()
'my_ABC_Class'
>>> WordSet.parse('a-command').camel_case()
'ACommand'
>>> WordSet.parse('someIdentifier').lowered().space_separated()
'some identifier'
Slices of the result should return another WordSet.
>>> WordSet.parse('taken-out-of-context')[1:].underscore_separated()
'out_of_context'
>>> WordSet.from_class_name(WordSet()).lowered().space_separated()
'word set'
>>> example = WordSet.parse('figured it out')
>>> example.headless_camel_case()
'figuredItOut'
>>> example.dash_separated()
'figured-it-out'
"""
_pattern = re.compile('([A-Z]?[a-z]+)|([A-Z]+(?![a-z]))')
def capitalized(self):
return WordSet(word.capitalize() for word in self)
def lowered(self):
return WordSet(word.lower() for word in self)
def camel_case(self):
return ''.join(self.capitalized())
def headless_camel_case(self):
words = iter(self)
first = next(words).lower()
new_words = itertools.chain((first,), WordSet(words).camel_case())
return ''.join(new_words)
def underscore_separated(self):
return '_'.join(self)
def dash_separated(self):
return '-'.join(self)
def space_separated(self):
return ' '.join(self)
def trim_right(self, item):
"""
Remove the item from the end of the set.
>>> WordSet.parse('foo bar').trim_right('foo')
('foo', 'bar')
>>> WordSet.parse('foo bar').trim_right('bar')
('foo',)
>>> WordSet.parse('').trim_right('bar')
()
"""
return self[:-1] if self and self[-1] == item else self
def trim_left(self, item):
"""
Remove the item from the beginning of the set.
>>> WordSet.parse('foo bar').trim_left('foo')
('bar',)
>>> WordSet.parse('foo bar').trim_left('bar')
('foo', 'bar')
>>> WordSet.parse('').trim_left('bar')
()
"""
return self[1:] if self and self[0] == item else self
def trim(self, item):
"""
>>> WordSet.parse('foo bar').trim('foo')
('bar',)
"""
return self.trim_left(item).trim_right(item)
def __getitem__(self, item):
result = super(WordSet, self).__getitem__(item)
if isinstance(item, slice):
result = WordSet(result)
return result
@classmethod
def parse(cls, identifier):
matches = cls._pattern.finditer(identifier)
return WordSet(match.group(0) for match in matches)
@classmethod
def from_class_name(cls, subject):
return cls.parse(subject.__class__.__name__)
# for backward compatibility
words = WordSet.parse
def simple_html_strip(s):
r"""
Remove HTML from the string `s`.
>>> str(simple_html_strip(''))
''
>>> print(simple_html_strip('A <bold>stormy</bold> day in paradise'))
A stormy day in paradise
>>> print(simple_html_strip('Somebody <!-- do not --> tell the truth.'))
Somebody tell the truth.
>>> print(simple_html_strip('What about<br/>\nmultiple lines?'))
What about
multiple lines?
"""
html_stripper = re.compile('(<!--.*?-->)|(<[^>]*>)|([^<]+)', re.DOTALL)
texts = (match.group(3) or '' for match in html_stripper.finditer(s))
return ''.join(texts)
class SeparatedValues(str):
"""
A string separated by a separator. Overrides __iter__ for getting
the values.
>>> list(SeparatedValues('a,b,c'))
['a', 'b', 'c']
Whitespace is stripped and empty values are discarded.
>>> list(SeparatedValues(' a, b , c, '))
['a', 'b', 'c']
"""
separator = ','
def __iter__(self):
parts = self.split(self.separator)
return filter(None, (part.strip() for part in parts))
class Stripper:
r"""
Given a series of lines, find the common prefix and strip it from them.
>>> lines = [
... 'abcdefg\n',
... 'abc\n',
... 'abcde\n',
... ]
>>> res = Stripper.strip_prefix(lines)
>>> res.prefix
'abc'
>>> list(res.lines)
['defg\n', '\n', 'de\n']
If no prefix is common, nothing should be stripped.
>>> lines = [
... 'abcd\n',
... '1234\n',
... ]
>>> res = Stripper.strip_prefix(lines)
>>> res.prefix = ''
>>> list(res.lines)
['abcd\n', '1234\n']
"""
def __init__(self, prefix, lines):
self.prefix = prefix
self.lines = map(self, lines)
@classmethod
def strip_prefix(cls, lines):
prefix_lines, lines = itertools.tee(lines)
prefix = functools.reduce(cls.common_prefix, prefix_lines)
return cls(prefix, lines)
def __call__(self, line):
if not self.prefix:
return line
null, prefix, rest = line.partition(self.prefix)
return rest
@staticmethod
def common_prefix(s1, s2):
"""
Return the common prefix of two lines.
"""
index = min(len(s1), len(s2))
while s1[:index] != s2[:index]:
index -= 1
return s1[:index]
def remove_prefix(text, prefix):
"""
Remove the prefix from the text if it exists.
>>> remove_prefix('underwhelming performance', 'underwhelming ')
'performance'
>>> remove_prefix('something special', 'sample')
'something special'
"""
null, prefix, rest = text.rpartition(prefix)
return rest
def remove_suffix(text, suffix):
"""
Remove the suffix from the text if it exists.
>>> remove_suffix('name.git', '.git')
'name'
>>> remove_suffix('something special', 'sample')
'something special'
"""
rest, suffix, null = text.partition(suffix)
return rest
def normalize_newlines(text):
r"""
Replace alternate newlines with the canonical newline.
>>> normalize_newlines('Lorem Ipsum\u2029')
'Lorem Ipsum\n'
>>> normalize_newlines('Lorem Ipsum\r\n')
'Lorem Ipsum\n'
>>> normalize_newlines('Lorem Ipsum\x85')
'Lorem Ipsum\n'
"""
newlines = ['\r\n', '\r', '\n', '\u0085', '\u2028', '\u2029']
pattern = '|'.join(newlines)
return re.sub(pattern, '\n', text)
def _nonblank(str):
return str and not str.startswith('#')
@functools.singledispatch
def yield_lines(iterable):
r"""
Yield valid lines of a string or iterable.
>>> list(yield_lines(''))
[]
>>> list(yield_lines(['foo', 'bar']))
['foo', 'bar']
>>> list(yield_lines('foo\nbar'))
['foo', 'bar']
>>> list(yield_lines('\nfoo\n#bar\nbaz #comment'))
['foo', 'baz #comment']
>>> list(yield_lines(['foo\nbar', 'baz', 'bing\n\n\n']))
['foo', 'bar', 'baz', 'bing']
"""
return itertools.chain.from_iterable(map(yield_lines, iterable))
@yield_lines.register(str)
def _(text):
return filter(_nonblank, map(str.strip, text.splitlines()))
def drop_comment(line):
"""
Drop comments.
>>> drop_comment('foo # bar')
'foo'
A hash without a space may be in a URL.
>>> drop_comment('http://example.com/foo#bar')
'http://example.com/foo#bar'
"""
return line.partition(' #')[0]
def join_continuation(lines):
r"""
Join lines continued by a trailing backslash.
>>> list(join_continuation(['foo \\', 'bar', 'baz']))
['foobar', 'baz']
>>> list(join_continuation(['foo \\', 'bar', 'baz']))
['foobar', 'baz']
>>> list(join_continuation(['foo \\', 'bar \\', 'baz']))
['foobarbaz']
Not sure why, but...
The character preceeding the backslash is also elided.
>>> list(join_continuation(['goo\\', 'dly']))
['godly']
A terrible idea, but...
If no line is available to continue, suppress the lines.
>>> list(join_continuation(['foo', 'bar\\', 'baz\\']))
['foo']
"""
lines = iter(lines)
for item in lines:
while item.endswith('\\'):
try:
item = item[:-2].strip() + next(lines)
except StopIteration:
return
yield item
@@ -2,6 +2,7 @@ import warnings
from collections import Counter, defaultdict, deque, abc
from collections.abc import Sequence
from concurrent.futures import ThreadPoolExecutor
from functools import partial, reduce, wraps
from heapq import merge, heapify, heapreplace, heappop
from itertools import (
@@ -3453,7 +3454,7 @@ class callback_iter:
self._aborted = False
self._future = None
self._wait_seconds = wait_seconds
self._executor = __import__("concurrent.futures").futures.ThreadPoolExecutor(max_workers=1)
self._executor = ThreadPoolExecutor(max_workers=1)
self._iterator = self._reader()
def __enter__(self):
@@ -1,145 +0,0 @@
import itertools
import functools
import contextlib
from setuptools.extern.packaging.requirements import Requirement
from setuptools.extern.packaging.version import Version
from setuptools.extern.more_itertools import always_iterable
from setuptools.extern.jaraco.context import suppress
from setuptools.extern.jaraco.functools import apply
from ._compat import metadata, repair_extras
def resolve(req: Requirement) -> metadata.Distribution:
"""
Resolve the requirement to its distribution.
Ignore exception detail for Python 3.9 compatibility.
>>> resolve(Requirement('pytest<3')) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
importlib.metadata.PackageNotFoundError: No package metadata was found for pytest<3
"""
dist = metadata.distribution(req.name)
if not req.specifier.contains(Version(dist.version), prereleases=True):
raise metadata.PackageNotFoundError(str(req))
dist.extras = req.extras # type: ignore
return dist
@apply(bool)
@suppress(metadata.PackageNotFoundError)
def is_satisfied(req: Requirement):
return resolve(req)
unsatisfied = functools.partial(itertools.filterfalse, is_satisfied)
class NullMarker:
@classmethod
def wrap(cls, req: Requirement):
return req.marker or cls()
def evaluate(self, *args, **kwargs):
return True
def find_direct_dependencies(dist, extras=None):
"""
Find direct, declared dependencies for dist.
"""
simple = (
req
for req in map(Requirement, always_iterable(dist.requires))
if NullMarker.wrap(req).evaluate(dict(extra=None))
)
extra_deps = (
req
for req in map(Requirement, always_iterable(dist.requires))
for extra in always_iterable(getattr(dist, 'extras', extras))
if NullMarker.wrap(req).evaluate(dict(extra=extra))
)
return itertools.chain(simple, extra_deps)
def traverse(items, visit):
"""
Given an iterable of items, traverse the items.
For each item, visit is called to return any additional items
to include in the traversal.
"""
while True:
try:
item = next(items)
except StopIteration:
return
yield item
items = itertools.chain(items, visit(item))
def find_req_dependencies(req):
with contextlib.suppress(metadata.PackageNotFoundError):
dist = resolve(req)
yield from find_direct_dependencies(dist)
def find_dependencies(dist, extras=None):
"""
Find all reachable dependencies for dist.
dist is an importlib.metadata.Distribution (or similar).
TODO: create a suitable protocol for type hint.
>>> deps = find_dependencies(resolve(Requirement('nspektr')))
>>> all(isinstance(dep, Requirement) for dep in deps)
True
>>> not any('pytest' in str(dep) for dep in deps)
True
>>> test_deps = find_dependencies(resolve(Requirement('nspektr[testing]')))
>>> any('pytest' in str(dep) for dep in test_deps)
True
"""
def visit(req, seen=set()):
if req in seen:
return ()
seen.add(req)
return find_req_dependencies(req)
return traverse(find_direct_dependencies(dist, extras), visit)
class Unresolved(Exception):
def __iter__(self):
return iter(self.args[0])
def missing(ep):
"""
Generate the unresolved dependencies (if any) of ep.
"""
return unsatisfied(find_dependencies(ep.dist, repair_extras(ep.extras)))
def check(ep):
"""
>>> ep, = metadata.entry_points(group='console_scripts', name='pip')
>>> check(ep)
>>> dist = metadata.distribution('nspektr')
Since 'docs' extras are not installed, requesting them should fail.
>>> ep = metadata.EntryPoint(
... group=None, name=None, value='nspektr [docs]')._for(dist)
>>> check(ep)
Traceback (most recent call last):
...
nspektr.Unresolved: [...]
"""
missed = list(missing(ep))
if missed:
raise Unresolved(missed)
@@ -1,21 +0,0 @@
import contextlib
import sys
if sys.version_info >= (3, 10):
import importlib.metadata as metadata
else:
import setuptools.extern.importlib_metadata as metadata # type: ignore # noqa: F401
def repair_extras(extras):
"""
Repair extras that appear as match objects.
python/importlib_metadata#369 revealed a flaw in the EntryPoint
implementation. This function wraps the extras to ensure
they are proper strings even on older implementations.
"""
with contextlib.suppress(AttributeError):
return list(item.group(0) for item in extras)
return extras
@@ -17,7 +17,7 @@ __title__ = "packaging"
__summary__ = "Core utilities for Python packages"
__uri__ = "https://github.com/pypa/packaging"
__version__ = "21.3"
__version__ = "21.2"
__author__ = "Donald Stufft and individual contributors"
__email__ = "donald@stufft.io"
@@ -98,7 +98,7 @@ def _get_musl_version(executable: str) -> Optional[_MuslVersion]:
with contextlib.ExitStack() as stack:
try:
f = stack.enter_context(open(executable, "rb"))
except OSError:
except IOError:
return None
ld = _parse_ld_musl_from_elf(f)
if not ld:
@@ -19,6 +19,9 @@ class InfinityType:
def __eq__(self, other: object) -> bool:
return isinstance(other, self.__class__)
def __ne__(self, other: object) -> bool:
return not isinstance(other, self.__class__)
def __gt__(self, other: object) -> bool:
return True
@@ -48,6 +51,9 @@ class NegativeInfinityType:
def __eq__(self, other: object) -> bool:
return isinstance(other, self.__class__)
def __ne__(self, other: object) -> bool:
return not isinstance(other, self.__class__)
def __gt__(self, other: object) -> bool:
return False
@@ -57,6 +57,13 @@ class BaseSpecifier(metaclass=abc.ABCMeta):
objects are equal.
"""
@abc.abstractmethod
def __ne__(self, other: object) -> bool:
"""
Returns a boolean representing whether or not the two Specifier like
objects are not equal.
"""
@abc.abstractproperty
def prereleases(self) -> Optional[bool]:
"""
@@ -112,7 +119,7 @@ class _IndividualSpecifier(BaseSpecifier):
else ""
)
return f"<{self.__class__.__name__}({str(self)!r}{pre})>"
return "<{}({!r}{})>".format(self.__class__.__name__, str(self), pre)
def __str__(self) -> str:
return "{}{}".format(*self._spec)
@@ -135,6 +142,17 @@ class _IndividualSpecifier(BaseSpecifier):
return self._canonical_spec == other._canonical_spec
def __ne__(self, other: object) -> bool:
if isinstance(other, str):
try:
other = self.__class__(str(other))
except InvalidSpecifier:
return NotImplemented
elif not isinstance(other, self.__class__):
return NotImplemented
return self._spec != other._spec
def _get_operator(self, op: str) -> CallableOperator:
operator_callable: CallableOperator = getattr(
self, f"_compare_{self._operators[op]}"
@@ -649,7 +667,7 @@ class SpecifierSet(BaseSpecifier):
else ""
)
return f"<SpecifierSet({str(self)!r}{pre})>"
return "<SpecifierSet({!r}{})>".format(str(self), pre)
def __str__(self) -> str:
return ",".join(sorted(str(s) for s in self._specs))
@@ -688,6 +706,14 @@ class SpecifierSet(BaseSpecifier):
return self._specs == other._specs
def __ne__(self, other: object) -> bool:
if isinstance(other, (str, _IndividualSpecifier)):
other = SpecifierSet(str(other))
elif not isinstance(other, SpecifierSet):
return NotImplemented
return self._specs != other._specs
def __len__(self) -> int:
return len(self._specs)
@@ -90,7 +90,7 @@ class Tag:
return f"{self._interpreter}-{self._abi}-{self._platform}"
def __repr__(self) -> str:
return f"<{self} @ {id(self)}>"
return "<{self} @ {self_id}>".format(self=self, self_id=id(self))
def parse_tag(tag: str) -> FrozenSet[Tag]:
@@ -192,7 +192,7 @@ def cpython_tags(
if not python_version:
python_version = sys.version_info[:2]
interpreter = f"cp{_version_nodot(python_version[:2])}"
interpreter = "cp{}".format(_version_nodot(python_version[:2]))
if abis is None:
if len(python_version) > 1:
@@ -268,11 +268,11 @@ def _py_interpreter_range(py_version: PythonVersion) -> Iterator[str]:
all previous versions of that major version.
"""
if len(py_version) > 1:
yield f"py{_version_nodot(py_version[:2])}"
yield f"py{py_version[0]}"
yield "py{version}".format(version=_version_nodot(py_version[:2]))
yield "py{major}".format(major=py_version[0])
if len(py_version) > 1:
for minor in range(py_version[1] - 1, -1, -1):
yield f"py{_version_nodot((py_version[0], minor))}"
yield "py{version}".format(version=_version_nodot((py_version[0], minor)))
def compatible_tags(
@@ -481,7 +481,4 @@ def sys_tags(*, warn: bool = False) -> Iterator[Tag]:
else:
yield from generic_tags()
if interp_name == "pp":
yield from compatible_tags(interpreter="pp3")
else:
yield from compatible_tags()
yield from compatible_tags()
@@ -1,11 +0,0 @@
# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
# Licensed to PSF under a Contributor Agreement.
__all__ = ("loads", "load", "TOMLDecodeError")
__version__ = "2.0.1" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT
from ._parser import TOMLDecodeError, load, loads
# Pretend this exception was created here.
TOMLDecodeError.__module__ = __name__
@@ -1,691 +0,0 @@
# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
# Licensed to PSF under a Contributor Agreement.
from __future__ import annotations
from collections.abc import Iterable
import string
from types import MappingProxyType
from typing import Any, BinaryIO, NamedTuple
from ._re import (
RE_DATETIME,
RE_LOCALTIME,
RE_NUMBER,
match_to_datetime,
match_to_localtime,
match_to_number,
)
from ._types import Key, ParseFloat, Pos
ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
# Neither of these sets include quotation mark or backslash. They are
# currently handled as separate cases in the parser functions.
ILLEGAL_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t")
ILLEGAL_MULTILINE_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t\n")
ILLEGAL_LITERAL_STR_CHARS = ILLEGAL_BASIC_STR_CHARS
ILLEGAL_MULTILINE_LITERAL_STR_CHARS = ILLEGAL_MULTILINE_BASIC_STR_CHARS
ILLEGAL_COMMENT_CHARS = ILLEGAL_BASIC_STR_CHARS
TOML_WS = frozenset(" \t")
TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n")
BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_")
KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'")
HEXDIGIT_CHARS = frozenset(string.hexdigits)
BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType(
{
"\\b": "\u0008", # backspace
"\\t": "\u0009", # tab
"\\n": "\u000A", # linefeed
"\\f": "\u000C", # form feed
"\\r": "\u000D", # carriage return
'\\"': "\u0022", # quote
"\\\\": "\u005C", # backslash
}
)
class TOMLDecodeError(ValueError):
"""An error raised if a document is not valid TOML."""
def load(__fp: BinaryIO, *, parse_float: ParseFloat = float) -> dict[str, Any]:
"""Parse TOML from a binary file object."""
b = __fp.read()
try:
s = b.decode()
except AttributeError:
raise TypeError(
"File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`"
) from None
return loads(s, parse_float=parse_float)
def loads(__s: str, *, parse_float: ParseFloat = float) -> dict[str, Any]: # noqa: C901
"""Parse TOML from a string."""
# The spec allows converting "\r\n" to "\n", even in string
# literals. Let's do so to simplify parsing.
src = __s.replace("\r\n", "\n")
pos = 0
out = Output(NestedDict(), Flags())
header: Key = ()
parse_float = make_safe_parse_float(parse_float)
# Parse one statement at a time
# (typically means one line in TOML source)
while True:
# 1. Skip line leading whitespace
pos = skip_chars(src, pos, TOML_WS)
# 2. Parse rules. Expect one of the following:
# - end of file
# - end of line
# - comment
# - key/value pair
# - append dict to list (and move to its namespace)
# - create dict (and move to its namespace)
# Skip trailing whitespace when applicable.
try:
char = src[pos]
except IndexError:
break
if char == "\n":
pos += 1
continue
if char in KEY_INITIAL_CHARS:
pos = key_value_rule(src, pos, out, header, parse_float)
pos = skip_chars(src, pos, TOML_WS)
elif char == "[":
try:
second_char: str | None = src[pos + 1]
except IndexError:
second_char = None
out.flags.finalize_pending()
if second_char == "[":
pos, header = create_list_rule(src, pos, out)
else:
pos, header = create_dict_rule(src, pos, out)
pos = skip_chars(src, pos, TOML_WS)
elif char != "#":
raise suffixed_err(src, pos, "Invalid statement")
# 3. Skip comment
pos = skip_comment(src, pos)
# 4. Expect end of line or end of file
try:
char = src[pos]
except IndexError:
break
if char != "\n":
raise suffixed_err(
src, pos, "Expected newline or end of document after a statement"
)
pos += 1
return out.data.dict
class Flags:
"""Flags that map to parsed keys/namespaces."""
# Marks an immutable namespace (inline array or inline table).
FROZEN = 0
# Marks a nest that has been explicitly created and can no longer
# be opened using the "[table]" syntax.
EXPLICIT_NEST = 1
def __init__(self) -> None:
self._flags: dict[str, dict] = {}
self._pending_flags: set[tuple[Key, int]] = set()
def add_pending(self, key: Key, flag: int) -> None:
self._pending_flags.add((key, flag))
def finalize_pending(self) -> None:
for key, flag in self._pending_flags:
self.set(key, flag, recursive=False)
self._pending_flags.clear()
def unset_all(self, key: Key) -> None:
cont = self._flags
for k in key[:-1]:
if k not in cont:
return
cont = cont[k]["nested"]
cont.pop(key[-1], None)
def set(self, key: Key, flag: int, *, recursive: bool) -> None: # noqa: A003
cont = self._flags
key_parent, key_stem = key[:-1], key[-1]
for k in key_parent:
if k not in cont:
cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}}
cont = cont[k]["nested"]
if key_stem not in cont:
cont[key_stem] = {"flags": set(), "recursive_flags": set(), "nested": {}}
cont[key_stem]["recursive_flags" if recursive else "flags"].add(flag)
def is_(self, key: Key, flag: int) -> bool:
if not key:
return False # document root has no flags
cont = self._flags
for k in key[:-1]:
if k not in cont:
return False
inner_cont = cont[k]
if flag in inner_cont["recursive_flags"]:
return True
cont = inner_cont["nested"]
key_stem = key[-1]
if key_stem in cont:
cont = cont[key_stem]
return flag in cont["flags"] or flag in cont["recursive_flags"]
return False
class NestedDict:
def __init__(self) -> None:
# The parsed content of the TOML document
self.dict: dict[str, Any] = {}
def get_or_create_nest(
self,
key: Key,
*,
access_lists: bool = True,
) -> dict:
cont: Any = self.dict
for k in key:
if k not in cont:
cont[k] = {}
cont = cont[k]
if access_lists and isinstance(cont, list):
cont = cont[-1]
if not isinstance(cont, dict):
raise KeyError("There is no nest behind this key")
return cont
def append_nest_to_list(self, key: Key) -> None:
cont = self.get_or_create_nest(key[:-1])
last_key = key[-1]
if last_key in cont:
list_ = cont[last_key]
if not isinstance(list_, list):
raise KeyError("An object other than list found behind this key")
list_.append({})
else:
cont[last_key] = [{}]
class Output(NamedTuple):
data: NestedDict
flags: Flags
def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos:
try:
while src[pos] in chars:
pos += 1
except IndexError:
pass
return pos
def skip_until(
src: str,
pos: Pos,
expect: str,
*,
error_on: frozenset[str],
error_on_eof: bool,
) -> Pos:
try:
new_pos = src.index(expect, pos)
except ValueError:
new_pos = len(src)
if error_on_eof:
raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None
if not error_on.isdisjoint(src[pos:new_pos]):
while src[pos] not in error_on:
pos += 1
raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}")
return new_pos
def skip_comment(src: str, pos: Pos) -> Pos:
try:
char: str | None = src[pos]
except IndexError:
char = None
if char == "#":
return skip_until(
src, pos + 1, "\n", error_on=ILLEGAL_COMMENT_CHARS, error_on_eof=False
)
return pos
def skip_comments_and_array_ws(src: str, pos: Pos) -> Pos:
while True:
pos_before_skip = pos
pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE)
pos = skip_comment(src, pos)
if pos == pos_before_skip:
return pos
def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]:
pos += 1 # Skip "["
pos = skip_chars(src, pos, TOML_WS)
pos, key = parse_key(src, pos)
if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN):
raise suffixed_err(src, pos, f"Cannot declare {key} twice")
out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False)
try:
out.data.get_or_create_nest(key)
except KeyError:
raise suffixed_err(src, pos, "Cannot overwrite a value") from None
if not src.startswith("]", pos):
raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
return pos + 1, key
def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]:
pos += 2 # Skip "[["
pos = skip_chars(src, pos, TOML_WS)
pos, key = parse_key(src, pos)
if out.flags.is_(key, Flags.FROZEN):
raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}")
# Free the namespace now that it points to another empty list item...
out.flags.unset_all(key)
# ...but this key precisely is still prohibited from table declaration
out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False)
try:
out.data.append_nest_to_list(key)
except KeyError:
raise suffixed_err(src, pos, "Cannot overwrite a value") from None
if not src.startswith("]]", pos):
raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
return pos + 2, key
def key_value_rule(
src: str, pos: Pos, out: Output, header: Key, parse_float: ParseFloat
) -> Pos:
pos, key, value = parse_key_value_pair(src, pos, parse_float)
key_parent, key_stem = key[:-1], key[-1]
abs_key_parent = header + key_parent
relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
for cont_key in relative_path_cont_keys:
# Check that dotted key syntax does not redefine an existing table
if out.flags.is_(cont_key, Flags.EXPLICIT_NEST):
raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}")
# Containers in the relative path can't be opened with the table syntax or
# dotted key/value syntax in following table sections.
out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST)
if out.flags.is_(abs_key_parent, Flags.FROZEN):
raise suffixed_err(
src, pos, f"Cannot mutate immutable namespace {abs_key_parent}"
)
try:
nest = out.data.get_or_create_nest(abs_key_parent)
except KeyError:
raise suffixed_err(src, pos, "Cannot overwrite a value") from None
if key_stem in nest:
raise suffixed_err(src, pos, "Cannot overwrite a value")
# Mark inline table and array namespaces recursively immutable
if isinstance(value, (dict, list)):
out.flags.set(header + key, Flags.FROZEN, recursive=True)
nest[key_stem] = value
return pos
def parse_key_value_pair(
src: str, pos: Pos, parse_float: ParseFloat
) -> tuple[Pos, Key, Any]:
pos, key = parse_key(src, pos)
try:
char: str | None = src[pos]
except IndexError:
char = None
if char != "=":
raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
pos += 1
pos = skip_chars(src, pos, TOML_WS)
pos, value = parse_value(src, pos, parse_float)
return pos, key, value
def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]:
pos, key_part = parse_key_part(src, pos)
key: Key = (key_part,)
pos = skip_chars(src, pos, TOML_WS)
while True:
try:
char: str | None = src[pos]
except IndexError:
char = None
if char != ".":
return pos, key
pos += 1
pos = skip_chars(src, pos, TOML_WS)
pos, key_part = parse_key_part(src, pos)
key += (key_part,)
pos = skip_chars(src, pos, TOML_WS)
def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]:
try:
char: str | None = src[pos]
except IndexError:
char = None
if char in BARE_KEY_CHARS:
start_pos = pos
pos = skip_chars(src, pos, BARE_KEY_CHARS)
return pos, src[start_pos:pos]
if char == "'":
return parse_literal_str(src, pos)
if char == '"':
return parse_one_line_basic_str(src, pos)
raise suffixed_err(src, pos, "Invalid initial character for a key part")
def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]:
pos += 1
return parse_basic_str(src, pos, multiline=False)
def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list]:
pos += 1
array: list = []
pos = skip_comments_and_array_ws(src, pos)
if src.startswith("]", pos):
return pos + 1, array
while True:
pos, val = parse_value(src, pos, parse_float)
array.append(val)
pos = skip_comments_and_array_ws(src, pos)
c = src[pos : pos + 1]
if c == "]":
return pos + 1, array
if c != ",":
raise suffixed_err(src, pos, "Unclosed array")
pos += 1
pos = skip_comments_and_array_ws(src, pos)
if src.startswith("]", pos):
return pos + 1, array
def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, dict]:
pos += 1
nested_dict = NestedDict()
flags = Flags()
pos = skip_chars(src, pos, TOML_WS)
if src.startswith("}", pos):
return pos + 1, nested_dict.dict
while True:
pos, key, value = parse_key_value_pair(src, pos, parse_float)
key_parent, key_stem = key[:-1], key[-1]
if flags.is_(key, Flags.FROZEN):
raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}")
try:
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
except KeyError:
raise suffixed_err(src, pos, "Cannot overwrite a value") from None
if key_stem in nest:
raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}")
nest[key_stem] = value
pos = skip_chars(src, pos, TOML_WS)
c = src[pos : pos + 1]
if c == "}":
return pos + 1, nested_dict.dict
if c != ",":
raise suffixed_err(src, pos, "Unclosed inline table")
if isinstance(value, (dict, list)):
flags.set(key, Flags.FROZEN, recursive=True)
pos += 1
pos = skip_chars(src, pos, TOML_WS)
def parse_basic_str_escape(
src: str, pos: Pos, *, multiline: bool = False
) -> tuple[Pos, str]:
escape_id = src[pos : pos + 2]
pos += 2
if multiline and escape_id in {"\\ ", "\\\t", "\\\n"}:
# Skip whitespace until next non-whitespace character or end of
# the doc. Error if non-whitespace is found before newline.
if escape_id != "\\\n":
pos = skip_chars(src, pos, TOML_WS)
try:
char = src[pos]
except IndexError:
return pos, ""
if char != "\n":
raise suffixed_err(src, pos, "Unescaped '\\' in a string")
pos += 1
pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE)
return pos, ""
if escape_id == "\\u":
return parse_hex_char(src, pos, 4)
if escape_id == "\\U":
return parse_hex_char(src, pos, 8)
try:
return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
except KeyError:
raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None
def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]:
return parse_basic_str_escape(src, pos, multiline=True)
def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]:
hex_str = src[pos : pos + hex_len]
if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str):
raise suffixed_err(src, pos, "Invalid hex value")
pos += hex_len
hex_int = int(hex_str, 16)
if not is_unicode_scalar_value(hex_int):
raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value")
return pos, chr(hex_int)
def parse_literal_str(src: str, pos: Pos) -> tuple[Pos, str]:
pos += 1 # Skip starting apostrophe
start_pos = pos
pos = skip_until(
src, pos, "'", error_on=ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True
)
return pos + 1, src[start_pos:pos] # Skip ending apostrophe
def parse_multiline_str(src: str, pos: Pos, *, literal: bool) -> tuple[Pos, str]:
pos += 3
if src.startswith("\n", pos):
pos += 1
if literal:
delim = "'"
end_pos = skip_until(
src,
pos,
"'''",
error_on=ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
error_on_eof=True,
)
result = src[pos:end_pos]
pos = end_pos + 3
else:
delim = '"'
pos, result = parse_basic_str(src, pos, multiline=True)
# Add at maximum two extra apostrophes/quotes if the end sequence
# is 4 or 5 chars long instead of just 3.
if not src.startswith(delim, pos):
return pos, result
pos += 1
if not src.startswith(delim, pos):
return pos, result + delim
pos += 1
return pos, result + (delim * 2)
def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]:
if multiline:
error_on = ILLEGAL_MULTILINE_BASIC_STR_CHARS
parse_escapes = parse_basic_str_escape_multiline
else:
error_on = ILLEGAL_BASIC_STR_CHARS
parse_escapes = parse_basic_str_escape
result = ""
start_pos = pos
while True:
try:
char = src[pos]
except IndexError:
raise suffixed_err(src, pos, "Unterminated string") from None
if char == '"':
if not multiline:
return pos + 1, result + src[start_pos:pos]
if src.startswith('"""', pos):
return pos + 3, result + src[start_pos:pos]
pos += 1
continue
if char == "\\":
result += src[start_pos:pos]
pos, parsed_escape = parse_escapes(src, pos)
result += parsed_escape
start_pos = pos
continue
if char in error_on:
raise suffixed_err(src, pos, f"Illegal character {char!r}")
pos += 1
def parse_value( # noqa: C901
src: str, pos: Pos, parse_float: ParseFloat
) -> tuple[Pos, Any]:
try:
char: str | None = src[pos]
except IndexError:
char = None
# IMPORTANT: order conditions based on speed of checking and likelihood
# Basic strings
if char == '"':
if src.startswith('"""', pos):
return parse_multiline_str(src, pos, literal=False)
return parse_one_line_basic_str(src, pos)
# Literal strings
if char == "'":
if src.startswith("'''", pos):
return parse_multiline_str(src, pos, literal=True)
return parse_literal_str(src, pos)
# Booleans
if char == "t":
if src.startswith("true", pos):
return pos + 4, True
if char == "f":
if src.startswith("false", pos):
return pos + 5, False
# Arrays
if char == "[":
return parse_array(src, pos, parse_float)
# Inline tables
if char == "{":
return parse_inline_table(src, pos, parse_float)
# Dates and times
datetime_match = RE_DATETIME.match(src, pos)
if datetime_match:
try:
datetime_obj = match_to_datetime(datetime_match)
except ValueError as e:
raise suffixed_err(src, pos, "Invalid date or datetime") from e
return datetime_match.end(), datetime_obj
localtime_match = RE_LOCALTIME.match(src, pos)
if localtime_match:
return localtime_match.end(), match_to_localtime(localtime_match)
# Integers and "normal" floats.
# The regex will greedily match any type starting with a decimal
# char, so needs to be located after handling of dates and times.
number_match = RE_NUMBER.match(src, pos)
if number_match:
return number_match.end(), match_to_number(number_match, parse_float)
# Special floats
first_three = src[pos : pos + 3]
if first_three in {"inf", "nan"}:
return pos + 3, parse_float(first_three)
first_four = src[pos : pos + 4]
if first_four in {"-inf", "+inf", "-nan", "+nan"}:
return pos + 4, parse_float(first_four)
raise suffixed_err(src, pos, "Invalid value")
def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError:
"""Return a `TOMLDecodeError` where error message is suffixed with
coordinates in source."""
def coord_repr(src: str, pos: Pos) -> str:
if pos >= len(src):
return "end of document"
line = src.count("\n", 0, pos) + 1
if line == 1:
column = pos + 1
else:
column = pos - src.rindex("\n", 0, pos)
return f"line {line}, column {column}"
return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})")
def is_unicode_scalar_value(codepoint: int) -> bool:
return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
def make_safe_parse_float(parse_float: ParseFloat) -> ParseFloat:
"""A decorator to make `parse_float` safe.
`parse_float` must not return dicts or lists, because these types
would be mixed with parsed TOML tables and arrays, thus confusing
the parser. The returned decorated callable raises `ValueError`
instead of returning illegal types.
"""
# The default `float` callable never returns illegal types. Optimize it.
if parse_float is float: # type: ignore[comparison-overlap]
return float
def safe_parse_float(float_str: str) -> Any:
float_value = parse_float(float_str)
if isinstance(float_value, (dict, list)):
raise ValueError("parse_float must not return dicts or lists")
return float_value
return safe_parse_float
@@ -1,107 +0,0 @@
# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
# Licensed to PSF under a Contributor Agreement.
from __future__ import annotations
from datetime import date, datetime, time, timedelta, timezone, tzinfo
from functools import lru_cache
import re
from typing import Any
from ._types import ParseFloat
# E.g.
# - 00:32:00.999999
# - 00:32:00
_TIME_RE_STR = r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?"
RE_NUMBER = re.compile(
r"""
0
(?:
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
b[01](?:_?[01])* # bin
|
o[0-7](?:_?[0-7])* # oct
)
|
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
(?P<floatpart>
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
)
""",
flags=re.VERBOSE,
)
RE_LOCALTIME = re.compile(_TIME_RE_STR)
RE_DATETIME = re.compile(
rf"""
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
(?:
[Tt ]
{_TIME_RE_STR}
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
)?
""",
flags=re.VERBOSE,
)
def match_to_datetime(match: re.Match) -> datetime | date:
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
Raises ValueError if the match does not correspond to a valid date
or datetime.
"""
(
year_str,
month_str,
day_str,
hour_str,
minute_str,
sec_str,
micros_str,
zulu_time,
offset_sign_str,
offset_hour_str,
offset_minute_str,
) = match.groups()
year, month, day = int(year_str), int(month_str), int(day_str)
if hour_str is None:
return date(year, month, day)
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
micros = int(micros_str.ljust(6, "0")) if micros_str else 0
if offset_sign_str:
tz: tzinfo | None = cached_tz(
offset_hour_str, offset_minute_str, offset_sign_str
)
elif zulu_time:
tz = timezone.utc
else: # local date-time
tz = None
return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
@lru_cache(maxsize=None)
def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone:
sign = 1 if sign_str == "+" else -1
return timezone(
timedelta(
hours=sign * int(hour_str),
minutes=sign * int(minute_str),
)
)
def match_to_localtime(match: re.Match) -> time:
hour_str, minute_str, sec_str, micros_str = match.groups()
micros = int(micros_str.ljust(6, "0")) if micros_str else 0
return time(int(hour_str), int(minute_str), int(sec_str), micros)
def match_to_number(match: re.Match, parse_float: ParseFloat) -> Any:
if match.group("floatpart"):
return parse_float(match.group())
return int(match.group(), 0)
@@ -1,10 +0,0 @@
# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
# Licensed to PSF under a Contributor Agreement.
from typing import Any, Callable, Tuple
# Type annotations
ParseFloat = Callable[[str], Any]
Key = Tuple[str, ...]
Pos = int
File diff suppressed because it is too large Load Diff
@@ -1,329 +0,0 @@
import io
import posixpath
import zipfile
import itertools
import contextlib
import sys
import pathlib
if sys.version_info < (3, 7):
from collections import OrderedDict
else:
OrderedDict = dict
__all__ = ['Path']
def _parents(path):
"""
Given a path with elements separated by
posixpath.sep, generate all parents of that path.
>>> list(_parents('b/d'))
['b']
>>> list(_parents('/b/d/'))
['/b']
>>> list(_parents('b/d/f/'))
['b/d', 'b']
>>> list(_parents('b'))
[]
>>> list(_parents(''))
[]
"""
return itertools.islice(_ancestry(path), 1, None)
def _ancestry(path):
"""
Given a path with elements separated by
posixpath.sep, generate all elements of that path
>>> list(_ancestry('b/d'))
['b/d', 'b']
>>> list(_ancestry('/b/d/'))
['/b/d', '/b']
>>> list(_ancestry('b/d/f/'))
['b/d/f', 'b/d', 'b']
>>> list(_ancestry('b'))
['b']
>>> list(_ancestry(''))
[]
"""
path = path.rstrip(posixpath.sep)
while path and path != posixpath.sep:
yield path
path, tail = posixpath.split(path)
_dedupe = OrderedDict.fromkeys
"""Deduplicate an iterable in original order"""
def _difference(minuend, subtrahend):
"""
Return items in minuend not in subtrahend, retaining order
with O(1) lookup.
"""
return itertools.filterfalse(set(subtrahend).__contains__, minuend)
class CompleteDirs(zipfile.ZipFile):
"""
A ZipFile subclass that ensures that implied directories
are always included in the namelist.
"""
@staticmethod
def _implied_dirs(names):
parents = itertools.chain.from_iterable(map(_parents, names))
as_dirs = (p + posixpath.sep for p in parents)
return _dedupe(_difference(as_dirs, names))
def namelist(self):
names = super(CompleteDirs, self).namelist()
return names + list(self._implied_dirs(names))
def _name_set(self):
return set(self.namelist())
def resolve_dir(self, name):
"""
If the name represents a directory, return that name
as a directory (with the trailing slash).
"""
names = self._name_set()
dirname = name + '/'
dir_match = name not in names and dirname in names
return dirname if dir_match else name
@classmethod
def make(cls, source):
"""
Given a source (filename or zipfile), return an
appropriate CompleteDirs subclass.
"""
if isinstance(source, CompleteDirs):
return source
if not isinstance(source, zipfile.ZipFile):
return cls(_pathlib_compat(source))
# Only allow for FastLookup when supplied zipfile is read-only
if 'r' not in source.mode:
cls = CompleteDirs
source.__class__ = cls
return source
class FastLookup(CompleteDirs):
"""
ZipFile subclass to ensure implicit
dirs exist and are resolved rapidly.
"""
def namelist(self):
with contextlib.suppress(AttributeError):
return self.__names
self.__names = super(FastLookup, self).namelist()
return self.__names
def _name_set(self):
with contextlib.suppress(AttributeError):
return self.__lookup
self.__lookup = super(FastLookup, self)._name_set()
return self.__lookup
def _pathlib_compat(path):
"""
For path-like objects, convert to a filename for compatibility
on Python 3.6.1 and earlier.
"""
try:
return path.__fspath__()
except AttributeError:
return str(path)
class Path:
"""
A pathlib-compatible interface for zip files.
Consider a zip file with this structure::
.
a.txt
b
c.txt
d
e.txt
>>> data = io.BytesIO()
>>> zf = zipfile.ZipFile(data, 'w')
>>> zf.writestr('a.txt', 'content of a')
>>> zf.writestr('b/c.txt', 'content of c')
>>> zf.writestr('b/d/e.txt', 'content of e')
>>> zf.filename = 'mem/abcde.zip'
Path accepts the zipfile object itself or a filename
>>> root = Path(zf)
From there, several path operations are available.
Directory iteration (including the zip file itself):
>>> a, b = root.iterdir()
>>> a
Path('mem/abcde.zip', 'a.txt')
>>> b
Path('mem/abcde.zip', 'b/')
name property:
>>> b.name
'b'
join with divide operator:
>>> c = b / 'c.txt'
>>> c
Path('mem/abcde.zip', 'b/c.txt')
>>> c.name
'c.txt'
Read text:
>>> c.read_text()
'content of c'
existence:
>>> c.exists()
True
>>> (b / 'missing.txt').exists()
False
Coercion to string:
>>> import os
>>> str(c).replace(os.sep, posixpath.sep)
'mem/abcde.zip/b/c.txt'
At the root, ``name``, ``filename``, and ``parent``
resolve to the zipfile. Note these attributes are not
valid and will raise a ``ValueError`` if the zipfile
has no filename.
>>> root.name
'abcde.zip'
>>> str(root.filename).replace(os.sep, posixpath.sep)
'mem/abcde.zip'
>>> str(root.parent)
'mem'
"""
__repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})"
def __init__(self, root, at=""):
"""
Construct a Path from a ZipFile or filename.
Note: When the source is an existing ZipFile object,
its type (__class__) will be mutated to a
specialized type. If the caller wishes to retain the
original type, the caller should either create a
separate ZipFile object or pass a filename.
"""
self.root = FastLookup.make(root)
self.at = at
def open(self, mode='r', *args, pwd=None, **kwargs):
"""
Open this entry as text or binary following the semantics
of ``pathlib.Path.open()`` by passing arguments through
to io.TextIOWrapper().
"""
if self.is_dir():
raise IsADirectoryError(self)
zip_mode = mode[0]
if not self.exists() and zip_mode == 'r':
raise FileNotFoundError(self)
stream = self.root.open(self.at, zip_mode, pwd=pwd)
if 'b' in mode:
if args or kwargs:
raise ValueError("encoding args invalid for binary operation")
return stream
return io.TextIOWrapper(stream, *args, **kwargs)
@property
def name(self):
return pathlib.Path(self.at).name or self.filename.name
@property
def suffix(self):
return pathlib.Path(self.at).suffix or self.filename.suffix
@property
def suffixes(self):
return pathlib.Path(self.at).suffixes or self.filename.suffixes
@property
def stem(self):
return pathlib.Path(self.at).stem or self.filename.stem
@property
def filename(self):
return pathlib.Path(self.root.filename).joinpath(self.at)
def read_text(self, *args, **kwargs):
with self.open('r', *args, **kwargs) as strm:
return strm.read()
def read_bytes(self):
with self.open('rb') as strm:
return strm.read()
def _is_child(self, path):
return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/")
def _next(self, at):
return self.__class__(self.root, at)
def is_dir(self):
return not self.at or self.at.endswith("/")
def is_file(self):
return self.exists() and not self.is_dir()
def exists(self):
return self.at in self.root._name_set()
def iterdir(self):
if not self.is_dir():
raise ValueError("Can't listdir a file")
subs = map(self._next, self.root.namelist())
return filter(self._is_child, subs)
def __str__(self):
return posixpath.join(self.root.filename, self.at)
def __repr__(self):
return self.__repr.format(self=self)
def joinpath(self, *other):
next = posixpath.join(self.at, *map(_pathlib_compat, other))
return self._next(self.root.resolve_dir(next))
__truediv__ = joinpath
@property
def parent(self):
if not self.at:
return self.filename.parent
parent_at = posixpath.dirname(self.at.rstrip('/'))
if parent_at:
parent_at += '/'
return self._next(parent_at)
@@ -8,7 +8,7 @@ import posixpath
import contextlib
from distutils.errors import DistutilsError
from ._path import ensure_directory
from pkg_resources import ensure_directory
__all__ = [
"unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter",
@@ -100,37 +100,29 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter):
raise UnrecognizedFormat("%s is not a zip file" % (filename,))
with zipfile.ZipFile(filename) as z:
_unpack_zipfile_obj(z, extract_dir, progress_filter)
for info in z.infolist():
name = info.filename
# don't extract absolute paths or ones with .. in them
if name.startswith('/') or '..' in name.split('/'):
continue
def _unpack_zipfile_obj(zipfile_obj, extract_dir, progress_filter=default_filter):
"""Internal/private API used by other parts of setuptools.
Similar to ``unpack_zipfile``, but receives an already opened :obj:`zipfile.ZipFile`
object instead of a filename.
"""
for info in zipfile_obj.infolist():
name = info.filename
# don't extract absolute paths or ones with .. in them
if name.startswith('/') or '..' in name.split('/'):
continue
target = os.path.join(extract_dir, *name.split('/'))
target = progress_filter(name, target)
if not target:
continue
if name.endswith('/'):
# directory
ensure_directory(target)
else:
# file
ensure_directory(target)
data = zipfile_obj.read(info.filename)
with open(target, 'wb') as f:
f.write(data)
unix_attributes = info.external_attr >> 16
if unix_attributes:
os.chmod(target, unix_attributes)
target = os.path.join(extract_dir, *name.split('/'))
target = progress_filter(name, target)
if not target:
continue
if name.endswith('/'):
# directory
ensure_directory(target)
else:
# file
ensure_directory(target)
data = z.read(info.filename)
with open(target, 'wb') as f:
f.write(data)
unix_attributes = info.external_attr >> 16
if unix_attributes:
os.chmod(target, unix_attributes)
def _resolve_tar_file_or_dir(tar_obj, tar_member_obj):
@@ -37,9 +37,8 @@ import warnings
import setuptools
import distutils
from ._reqs import parse_strings
from .extern.more_itertools import always_iterable
from pkg_resources import parse_requirements
__all__ = ['get_requires_for_build_sdist',
'get_requires_for_build_wheel',
@@ -57,7 +56,7 @@ class SetupRequirementsError(BaseException):
class Distribution(setuptools.dist.Distribution):
def fetch_build_eggs(self, specifiers):
specifier_list = list(parse_strings(specifiers))
specifier_list = list(map(str, parse_requirements(specifiers)))
raise SetupRequirementsError(specifier_list)
@@ -127,26 +126,11 @@ def suppress_known_deprecation():
yield
class _BuildMetaBackend:
class _BuildMetaBackend(object):
@staticmethod
def _fix_config(config_settings):
"""
Ensure config settings meet certain expectations.
>>> fc = _BuildMetaBackend._fix_config
>>> fc(None)
{'--global-option': []}
>>> fc({})
{'--global-option': []}
>>> fc({'--global-option': 'foo'})
{'--global-option': ['foo']}
>>> fc({'--global-option': ['foo']})
{'--global-option': ['foo']}
"""
def _fix_config(self, config_settings):
config_settings = config_settings or {}
config_settings['--global-option'] = list(always_iterable(
config_settings.get('--global-option')))
config_settings.setdefault('--global-option', [])
return config_settings
def _get_build_requires(self, config_settings, requirements):
@@ -174,10 +158,12 @@ class _BuildMetaBackend:
exec(compile(code, __file__, 'exec'), locals())
def get_requires_for_build_wheel(self, config_settings=None):
config_settings = self._fix_config(config_settings)
return self._get_build_requires(
config_settings, requirements=['wheel'])
def get_requires_for_build_sdist(self, config_settings=None):
config_settings = self._fix_config(config_settings)
return self._get_build_requires(config_settings, requirements=[])
def prepare_metadata_for_build_wheel(self, metadata_directory,
@@ -11,10 +11,9 @@ import re
import textwrap
import marshal
from pkg_resources import get_build_platform, Distribution
from pkg_resources import get_build_platform, Distribution, ensure_directory
from setuptools.extension import Library
from setuptools import Command
from .._path import ensure_directory
from sysconfig import get_path, get_python_version
@@ -4,13 +4,9 @@ As defined in the wheel specification
"""
import os
import re
import warnings
from inspect import cleandoc
from distutils.core import Command
from distutils import log
from setuptools.extern import packaging
class dist_info(Command):
@@ -33,36 +29,8 @@ class dist_info(Command):
egg_info.egg_base = self.egg_base
egg_info.finalize_options()
egg_info.run()
name = _safe(self.distribution.get_name())
version = _version(self.distribution.get_version())
base = self.egg_base or os.curdir
dist_info_dir = os.path.join(base, f"{name}-{version}.dist-info")
dist_info_dir = egg_info.egg_info[:-len('.egg-info')] + '.dist-info'
log.info("creating '{}'".format(os.path.abspath(dist_info_dir)))
bdist_wheel = self.get_finalized_command('bdist_wheel')
bdist_wheel.egg2dist(egg_info.egg_info, dist_info_dir)
def _safe(component: str) -> str:
"""Escape a component used to form a wheel name according to PEP 491"""
return re.sub(r"[^\w\d.]+", "_", component)
def _version(version: str) -> str:
"""Convert an arbitrary string to a version string."""
v = version.replace(' ', '.')
try:
return str(packaging.version.Version(v)).replace("-", "_")
except packaging.version.InvalidVersion:
msg = f"""!!\n\n
###################
# Invalid version #
###################
{version!r} is not valid according to PEP 440.\n
Please make sure specify a valid version for your package.
Also note that future releases of setuptools may halt the build process
if an invalid version is given.
\n\n!!
"""
warnings.warn(cleandoc(msg))
return _safe(v).strip("_")
@@ -39,10 +39,9 @@ import subprocess
import shlex
import io
import configparser
import sysconfig
from sysconfig import get_path
from sysconfig import get_config_vars, get_path
from setuptools import SetuptoolsDeprecationWarning
@@ -56,21 +55,18 @@ from setuptools.package_index import (
from setuptools.command import bdist_egg, egg_info
from setuptools.wheel import Wheel
from pkg_resources import (
normalize_path, resource_string,
yield_lines, normalize_path, resource_string, ensure_directory,
get_distribution, find_distributions, Environment, Requirement,
Distribution, PathMetadata, EggMetadata, WorkingSet, DistributionNotFound,
VersionConflict, DEVELOP_DIST,
)
import pkg_resources
from .._path import ensure_directory
from ..extern.jaraco.text import yield_lines
# Turn on PEP440Warnings
warnings.filterwarnings("default", category=pkg_resources.PEP440Warning)
__all__ = [
'easy_install', 'PthDistributions', 'extract_wininst_cfg',
'samefile', 'easy_install', 'PthDistributions', 'extract_wininst_cfg',
'get_exe_prefixes',
]
@@ -79,6 +75,22 @@ def is_64bit():
return struct.calcsize("P") == 8
def samefile(p1, p2):
"""
Determine if two paths reference the same file.
Augments os.path.samefile to work on Windows and
suppresses errors if the path doesn't exist.
"""
both_exist = os.path.exists(p1) and os.path.exists(p2)
use_samefile = hasattr(os.path, 'samefile') and both_exist
if use_samefile:
return os.path.samefile(p1, p2)
norm_p1 = os.path.normpath(os.path.normcase(p1))
norm_p2 = os.path.normpath(os.path.normcase(p2))
return norm_p1 == norm_p2
def _to_bytes(s):
return s.encode('utf8')
@@ -169,8 +181,12 @@ class easy_install(Command):
self.install_data = None
self.install_base = None
self.install_platbase = None
self.install_userbase = site.USER_BASE
self.install_usersite = site.USER_SITE
if site.ENABLE_USER_SITE:
self.install_userbase = site.USER_BASE
self.install_usersite = site.USER_SITE
else:
self.install_userbase = None
self.install_usersite = None
self.no_find_links = None
# Options not specifiable via command line
@@ -220,22 +236,23 @@ class easy_install(Command):
self.version and self._render_version()
py_version = sys.version.split()[0]
prefix, exec_prefix = get_config_vars('prefix', 'exec_prefix')
self.config_vars = dict(sysconfig.get_config_vars())
self.config_vars.update({
self.config_vars = {
'dist_name': self.distribution.get_name(),
'dist_version': self.distribution.get_version(),
'dist_fullname': self.distribution.get_fullname(),
'py_version': py_version,
'py_version_short': f'{sys.version_info.major}.{sys.version_info.minor}',
'py_version_nodot': f'{sys.version_info.major}{sys.version_info.minor}',
'sys_prefix': self.config_vars['prefix'],
'sys_exec_prefix': self.config_vars['exec_prefix'],
'py_version_short': py_version[0:3],
'py_version_nodot': py_version[0] + py_version[2],
'sys_prefix': prefix,
'prefix': prefix,
'sys_exec_prefix': exec_prefix,
'exec_prefix': exec_prefix,
# Only python 3.2+ has abiflags
'abiflags': getattr(sys, 'abiflags', ''),
'platlibdir': getattr(sys, 'platlibdir', 'lib'),
})
}
with contextlib.suppress(AttributeError):
# only for distutils outside stdlib
self.config_vars.update({
@@ -243,15 +260,11 @@ class easy_install(Command):
'implementation': install._get_implementation(),
})
# pypa/distutils#113 Python 3.9 compat
self.config_vars.setdefault(
'py_version_nodot_plat',
getattr(sys, 'windir', '').replace('.', ''),
)
if site.ENABLE_USER_SITE:
self.config_vars['userbase'] = self.install_userbase
self.config_vars['usersite'] = self.install_usersite
self.config_vars['userbase'] = self.install_userbase
self.config_vars['usersite'] = self.install_usersite
if self.user and not site.ENABLE_USER_SITE:
elif self.user:
log.warn("WARNING: The user site-packages directory is disabled.")
self._fix_install_dir_for_user_site()
@@ -287,14 +300,27 @@ class easy_install(Command):
self.script_dir = self.install_scripts
# default --record from the install command
self.set_undefined_options('install', ('record', 'record'))
# Should this be moved to the if statement below? It's not used
# elsewhere
normpath = map(normalize_path, sys.path)
self.all_site_dirs = get_site_dirs()
self.all_site_dirs.extend(self._process_site_dirs(self.site_dirs))
if self.site_dirs is not None:
site_dirs = [
os.path.expanduser(s.strip()) for s in
self.site_dirs.split(',')
]
for d in site_dirs:
if not os.path.isdir(d):
log.warn("%s (in --site-dirs) does not exist", d)
elif normalize_path(d) not in normpath:
raise DistutilsOptionError(
d + " (in --site-dirs) is not on sys.path"
)
else:
self.all_site_dirs.append(normalize_path(d))
if not self.editable:
self.check_site_dir()
default_index = os.getenv("__EASYINSTALL_INDEX", "https://pypi.org/simple/")
# ^ Private API for testing purposes only
self.index_url = self.index_url or default_index
self.index_url = self.index_url or "https://pypi.org/simple/"
self.shadow_path = self.all_site_dirs[:]
for path_item in self.install_dir, normalize_path(self.script_dir):
if path_item not in self.shadow_path:
@@ -320,7 +346,15 @@ class easy_install(Command):
if not self.no_find_links:
self.package_index.add_find_links(self.find_links)
self.set_undefined_options('install_lib', ('optimize', 'optimize'))
self.optimize = self._validate_optimize(self.optimize)
if not isinstance(self.optimize, int):
try:
self.optimize = int(self.optimize)
if not (0 <= self.optimize <= 2):
raise ValueError
except ValueError as e:
raise DistutilsOptionError(
"--optimize must be 0, 1, or 2"
) from e
if self.editable and not self.build_directory:
raise DistutilsArgError(
@@ -332,44 +366,11 @@ class easy_install(Command):
self.outputs = []
@staticmethod
def _process_site_dirs(site_dirs):
if site_dirs is None:
return
normpath = map(normalize_path, sys.path)
site_dirs = [
os.path.expanduser(s.strip()) for s in
site_dirs.split(',')
]
for d in site_dirs:
if not os.path.isdir(d):
log.warn("%s (in --site-dirs) does not exist", d)
elif normalize_path(d) not in normpath:
raise DistutilsOptionError(
d + " (in --site-dirs) is not on sys.path"
)
else:
yield normalize_path(d)
@staticmethod
def _validate_optimize(value):
try:
value = int(value)
if value not in range(3):
raise ValueError
except ValueError as e:
raise DistutilsOptionError(
"--optimize must be 0, 1, or 2"
) from e
return value
def _fix_install_dir_for_user_site(self):
"""
Fix the install_dir if "--user" was used.
"""
if not self.user:
if not self.user or not site.ENABLE_USER_SITE:
return
self.create_home_path()
@@ -918,9 +919,7 @@ class easy_install(Command):
ensure_directory(destination)
dist = self.egg_distribution(egg_path)
if not (
os.path.exists(destination) and os.path.samefile(egg_path, destination)
):
if not samefile(egg_path, destination):
if os.path.isdir(destination) and not os.path.islink(destination):
dir_util.remove_tree(destination, dry_run=self.dry_run)
elif os.path.exists(destination):
@@ -1329,7 +1328,7 @@ class easy_install(Command):
if not self.user:
return
home = convert_path(os.path.expanduser("~"))
for path in only_strs(self.config_vars.values()):
for name, path in self.config_vars.items():
if path.startswith(home) and not os.path.isdir(path):
self.debug_print("os.makedirs('%s', 0o700)" % path)
os.makedirs(path, 0o700)
@@ -1351,7 +1350,7 @@ class easy_install(Command):
if self.prefix:
# Set default install_dir/scripts from --prefix
config_vars = dict(config_vars)
config_vars = config_vars.copy()
config_vars['base'] = self.prefix
scheme = self.INSTALL_SCHEMES.get(os.name, self.DEFAULT_SCHEME)
for attr, val in scheme.items():
@@ -1578,7 +1577,7 @@ class PthDistributions(Environment):
self.sitedirs = list(map(normalize_path, sitedirs))
self.basedir = normalize_path(os.path.dirname(self.filename))
self._load()
super().__init__([], None, None)
Environment.__init__(self, [], None, None)
for path in yield_lines(self.paths):
list(map(self.add, find_distributions(path, True)))
@@ -1651,14 +1650,14 @@ class PthDistributions(Environment):
if new_path:
self.paths.append(dist.location)
self.dirty = True
super().add(dist)
Environment.add(self, dist)
def remove(self, dist):
"""Remove `dist` from the distribution map"""
while dist.location in self.paths:
self.paths.remove(dist.location)
self.dirty = True
super().remove(dist)
Environment.remove(self, dist)
def make_relative(self, path):
npath, last = os.path.split(normalize_path(path))
@@ -2299,13 +2298,6 @@ def current_umask():
return tmp
def only_strs(values):
"""
Exclude non-str values. Ref #3063.
"""
return filter(lambda val: isinstance(val, str), values)
class EasyInstallDeprecationWarning(SetuptoolsDeprecationWarning):
"""
Warning for EasyInstall deprecations, bypassing suppression.
@@ -17,22 +17,18 @@ import warnings
import time
import collections
from .._importlib import metadata
from .. import _entry_points
from setuptools import Command
from setuptools.command.sdist import sdist
from setuptools.command.sdist import walk_revctrl
from setuptools.command.setopt import edit_config
from setuptools.command import bdist_egg
from pkg_resources import (
Requirement, safe_name, parse_version,
safe_version, to_filename)
parse_requirements, safe_name, parse_version,
safe_version, yield_lines, EntryPoint, iter_entry_points, to_filename)
import setuptools.unicode_utils as unicode_utils
from setuptools.glob import glob
from setuptools.extern import packaging
from setuptools.extern.jaraco.text import yield_lines
from setuptools import SetuptoolsDeprecationWarning
@@ -136,21 +132,11 @@ class InfoCommon:
in which case the version string already contains all tags.
"""
return (
version if self.vtags and self._already_tagged(version)
version if self.vtags and version.endswith(self.vtags)
else version + self.vtags
)
def _already_tagged(self, version: str) -> bool:
# Depending on their format, tags may change with version normalization.
# So in addition the regular tags, we have to search for the normalized ones.
return version.endswith(self.vtags) or version.endswith(self._safe_tags())
def _safe_tags(self) -> str:
# To implement this we can rely on `safe_version` pretending to be version 0
# followed by tags. Then we simply discard the starting 0 (fake version number)
return safe_version(f"0{self.vtags}")[1:]
def tags(self) -> str:
def tags(self):
version = ''
if self.tag_build:
version += self.tag_build
@@ -219,8 +205,12 @@ class egg_info(InfoCommon, Command):
try:
is_version = isinstance(parsed_version, packaging.version.Version)
spec = "%s==%s" if is_version else "%s===%s"
Requirement(spec % (self.egg_name, self.egg_version))
spec = (
"%s==%s" if is_version else "%s===%s"
)
list(
parse_requirements(spec % (self.egg_name, self.egg_version))
)
except ValueError as e:
raise distutils.errors.DistutilsOptionError(
"Invalid distribution name or version syntax: %s-%s" %
@@ -295,9 +285,10 @@ class egg_info(InfoCommon, Command):
def run(self):
self.mkpath(self.egg_info)
os.utime(self.egg_info, None)
for ep in metadata.entry_points(group='egg_info.writers'):
self.distribution._install_dependencies(ep)
writer = ep.load()
installer = self.distribution.fetch_build_egg
for ep in iter_entry_points('egg_info.writers'):
ep.require(installer=installer)
writer = ep.resolve()
writer(self, ep.name, os.path.join(self.egg_info, ep.name))
# Get rid of native_libs.txt if it was put there by older bdist_egg
@@ -728,9 +719,20 @@ def write_arg(cmd, basename, filename, force=False):
def write_entries(cmd, basename, filename):
eps = _entry_points.load(cmd.distribution.entry_points)
defn = _entry_points.render(eps)
cmd.write_or_delete_file('entry points', filename, defn, True)
ep = cmd.distribution.entry_points
if isinstance(ep, str) or ep is None:
data = ep
elif ep is not None:
data = []
for section, contents in sorted(ep.items()):
if not isinstance(contents, str):
contents = EntryPoint.parse_group(section, contents)
contents = '\n'.join(sorted(map(str, contents.values())))
data.append('[%s]\n%s\n\n' % (section, contents))
data = ''.join(data)
cmd.write_or_delete_file('entry points', filename, data, True)
def get_pkg_info_revision():
@@ -91,21 +91,14 @@ class install(orig.install):
msg = "For best results, pass -X:Frames to enable call stack."
warnings.warn(msg)
return True
frames = inspect.getouterframes(run_frame)
for frame in frames[2:4]:
caller, = frame[:1]
info = inspect.getframeinfo(caller)
caller_module = caller.f_globals.get('__name__', '')
if caller_module == "setuptools.dist" and info.function == "run_command":
# Starting from v61.0.0 setuptools overwrites dist.run_command
continue
return (
caller_module == 'distutils.dist'
and info.function == 'run_commands'
)
res = inspect.getouterframes(run_frame)[2]
caller, = res[:1]
info = inspect.getframeinfo(caller)
caller_module = caller.f_globals.get('__name__', '')
return (
caller_module == 'distutils.dist'
and info.function == 'run_commands'
)
def do_egg_install(self):
@@ -4,7 +4,6 @@ import os
from setuptools import Command
from setuptools import namespaces
from setuptools.archive_util import unpack_archive
from .._path import ensure_directory
import pkg_resources
@@ -38,7 +37,7 @@ class install_egg_info(namespaces.Installer, Command):
elif os.path.exists(self.target):
self.execute(os.unlink, (self.target,), "Removing " + self.target)
if not self.dry_run:
ensure_directory(self.target)
pkg_resources.ensure_directory(self.target)
self.execute(
self.copytree, (), "Copying %s to %s" % (self.source, self.target)
)
@@ -4,8 +4,7 @@ from distutils.errors import DistutilsModuleError
import os
import sys
from pkg_resources import Distribution, PathMetadata
from .._path import ensure_directory
from pkg_resources import Distribution, PathMetadata, ensure_directory
class install_scripts(orig.install_scripts):
@@ -7,14 +7,14 @@ import contextlib
from .py36compat import sdist_add_defaults
from .._importlib import metadata
import pkg_resources
_default_revctrl = list
def walk_revctrl(dirname=''):
"""Find all files under revision control"""
for ep in metadata.entry_points(group='setuptools.file_finders'):
for ep in pkg_resources.iter_entry_points('setuptools.file_finders'):
for item in ep.load()(dirname):
yield item
@@ -16,11 +16,10 @@ from pkg_resources import (
evaluate_marker,
add_activation_listener,
require,
EntryPoint,
)
from .._importlib import metadata
from setuptools import Command
from setuptools.extern.more_itertools import unique_everseen
from setuptools.extern.jaraco.functools import pass_none
class ScanningLoader(TestLoader):
@@ -242,10 +241,12 @@ class test(Command):
return ['unittest'] + self.test_args
@staticmethod
@pass_none
def _resolve_as_ep(val):
"""
Load the indicated attribute value, called, as a as if it were
specified as an entry point.
"""
return metadata.EntryPoint(value=val, name=None, group=None).load()()
if val is None:
return
parsed = EntryPoint.parse("x=" + val)
return parsed.resolve()()
@@ -17,11 +17,8 @@ import itertools
import functools
import http.client
import urllib.parse
import warnings
from .._importlib import metadata
from .. import SetuptoolsDeprecationWarning
from pkg_resources import iter_entry_points
from .upload import upload
@@ -46,10 +43,9 @@ class upload_docs(upload):
boolean_options = upload.boolean_options
def has_sphinx(self):
return bool(
self.upload_dir is None
and metadata.entry_points(group='distutils.commands', name='build_sphinx')
)
if self.upload_dir is None:
for ep in iter_entry_points('distutils.commands', 'build_sphinx'):
return True
sub_commands = [('build_sphinx', has_sphinx)]
@@ -91,12 +87,6 @@ class upload_docs(upload):
zip_file.close()
def run(self):
warnings.warn(
"upload_docs is deprecated and will be removed in a future "
"version. Use tools like httpie or curl instead.",
SetuptoolsDeprecationWarning,
)
# Run sub commands
for cmd_name in self.get_sub_commands():
self.run_command(cmd_name)
@@ -1,35 +0,0 @@
"""For backward compatibility, expose main functions from
``setuptools.config.setupcfg``
"""
import warnings
from functools import wraps
from textwrap import dedent
from typing import Callable, TypeVar, cast
from .._deprecation_warning import SetuptoolsDeprecationWarning
from . import setupcfg
Fn = TypeVar("Fn", bound=Callable)
__all__ = ('parse_configuration', 'read_configuration')
def _deprecation_notice(fn: Fn) -> Fn:
@wraps(fn)
def _wrapper(*args, **kwargs):
msg = f"""\
As setuptools moves its configuration towards `pyproject.toml`,
`{__name__}.{fn.__name__}` became deprecated.
For the time being, you can use the `{setupcfg.__name__}` module
to access a backward compatible API, but this module is provisional
and might be removed in the future.
"""
warnings.warn(dedent(msg), SetuptoolsDeprecationWarning)
return fn(*args, **kwargs)
return cast(Fn, _wrapper)
read_configuration = _deprecation_notice(setupcfg.read_configuration)
parse_configuration = _deprecation_notice(setupcfg.parse_configuration)
@@ -1,374 +0,0 @@
"""Translation layer between pyproject config and setuptools distribution and
metadata objects.
The distribution and metadata objects are modeled after (an old version of)
core metadata, therefore configs in the format specified for ``pyproject.toml``
need to be processed before being applied.
"""
import logging
import os
import warnings
from collections.abc import Mapping
from email.headerregistry import Address
from functools import partial, reduce
from itertools import chain
from types import MappingProxyType
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
Type, Union)
if TYPE_CHECKING:
from setuptools._importlib import metadata # noqa
from setuptools.dist import Distribution # noqa
EMPTY: Mapping = MappingProxyType({}) # Immutable dict-like
_Path = Union[os.PathLike, str]
_DictOrStr = Union[dict, str]
_CorrespFn = Callable[["Distribution", Any, _Path], None]
_Correspondence = Union[str, _CorrespFn]
_logger = logging.getLogger(__name__)
def apply(dist: "Distribution", config: dict, filename: _Path) -> "Distribution":
"""Apply configuration dict read with :func:`read_configuration`"""
if not config:
return dist # short-circuit unrelated pyproject.toml file
root_dir = os.path.dirname(filename) or "."
_apply_project_table(dist, config, root_dir)
_apply_tool_table(dist, config, filename)
current_directory = os.getcwd()
os.chdir(root_dir)
try:
dist._finalize_requires()
dist._finalize_license_files()
finally:
os.chdir(current_directory)
return dist
def _apply_project_table(dist: "Distribution", config: dict, root_dir: _Path):
project_table = config.get("project", {}).copy()
if not project_table:
return # short-circuit
_handle_missing_dynamic(dist, project_table)
_unify_entry_points(project_table)
for field, value in project_table.items():
norm_key = json_compatible_key(field)
corresp = PYPROJECT_CORRESPONDENCE.get(norm_key, norm_key)
if callable(corresp):
corresp(dist, value, root_dir)
else:
_set_config(dist, corresp, value)
def _apply_tool_table(dist: "Distribution", config: dict, filename: _Path):
tool_table = config.get("tool", {}).get("setuptools", {})
if not tool_table:
return # short-circuit
for field, value in tool_table.items():
norm_key = json_compatible_key(field)
norm_key = TOOL_TABLE_RENAMES.get(norm_key, norm_key)
_set_config(dist, norm_key, value)
_copy_command_options(config, dist, filename)
def _handle_missing_dynamic(dist: "Distribution", project_table: dict):
"""Be temporarily forgiving with ``dynamic`` fields not listed in ``dynamic``"""
# TODO: Set fields back to `None` once the feature stabilizes
dynamic = set(project_table.get("dynamic", []))
for field, getter in _PREVIOUSLY_DEFINED.items():
if not (field in project_table or field in dynamic):
value = getter(dist)
if value:
msg = _WouldIgnoreField.message(field, value)
warnings.warn(msg, _WouldIgnoreField)
def json_compatible_key(key: str) -> str:
"""As defined in :pep:`566#json-compatible-metadata`"""
return key.lower().replace("-", "_")
def _set_config(dist: "Distribution", field: str, value: Any):
setter = getattr(dist.metadata, f"set_{field}", None)
if setter:
setter(value)
elif hasattr(dist.metadata, field) or field in SETUPTOOLS_PATCHES:
setattr(dist.metadata, field, value)
else:
setattr(dist, field, value)
_CONTENT_TYPES = {
".md": "text/markdown",
".rst": "text/x-rst",
".txt": "text/plain",
}
def _guess_content_type(file: str) -> Optional[str]:
_, ext = os.path.splitext(file.lower())
if not ext:
return None
if ext in _CONTENT_TYPES:
return _CONTENT_TYPES[ext]
valid = ", ".join(f"{k} ({v})" for k, v in _CONTENT_TYPES.items())
msg = f"only the following file extensions are recognized: {valid}."
raise ValueError(f"Undefined content type for {file}, {msg}")
def _long_description(dist: "Distribution", val: _DictOrStr, root_dir: _Path):
from setuptools.config import expand
if isinstance(val, str):
text = expand.read_files(val, root_dir)
ctype = _guess_content_type(val)
else:
text = val.get("text") or expand.read_files(val.get("file", []), root_dir)
ctype = val["content-type"]
_set_config(dist, "long_description", text)
if ctype:
_set_config(dist, "long_description_content_type", ctype)
def _license(dist: "Distribution", val: dict, root_dir: _Path):
from setuptools.config import expand
if "file" in val:
_set_config(dist, "license", expand.read_files([val["file"]], root_dir))
else:
_set_config(dist, "license", val["text"])
def _people(dist: "Distribution", val: List[dict], _root_dir: _Path, kind: str):
field = []
email_field = []
for person in val:
if "name" not in person:
email_field.append(person["email"])
elif "email" not in person:
field.append(person["name"])
else:
addr = Address(display_name=person["name"], addr_spec=person["email"])
email_field.append(str(addr))
if field:
_set_config(dist, kind, ", ".join(field))
if email_field:
_set_config(dist, f"{kind}_email", ", ".join(email_field))
def _project_urls(dist: "Distribution", val: dict, _root_dir):
special = {"downloadurl": "download_url", "homepage": "url"}
for key, url in val.items():
norm_key = json_compatible_key(key).replace("_", "")
_set_config(dist, special.get(norm_key, key), url)
# If `homepage` is missing, distutils will warn the following message:
# "warning: check: missing required meta-data: url"
# In the context of PEP 621, users might ask themselves: "which url?".
# Let's add a warning before distutils check to help users understand the problem:
if not dist.metadata.url:
msg = (
"Missing `Homepage` url.\nIt is advisable to link some kind of reference "
"for your project (e.g. source code or documentation).\n"
)
_logger.warning(msg)
_set_config(dist, "project_urls", val.copy())
def _python_requires(dist: "Distribution", val: dict, _root_dir):
from setuptools.extern.packaging.specifiers import SpecifierSet
_set_config(dist, "python_requires", SpecifierSet(val))
def _dependencies(dist: "Distribution", val: list, _root_dir):
existing = getattr(dist, "install_requires", [])
_set_config(dist, "install_requires", existing + val)
def _optional_dependencies(dist: "Distribution", val: dict, _root_dir):
existing = getattr(dist, "extras_require", {})
_set_config(dist, "extras_require", {**existing, **val})
def _unify_entry_points(project_table: dict):
project = project_table
entry_points = project.pop("entry-points", project.pop("entry_points", {}))
renaming = {"scripts": "console_scripts", "gui_scripts": "gui_scripts"}
for key, value in list(project.items()): # eager to allow modifications
norm_key = json_compatible_key(key)
if norm_key in renaming and value:
entry_points[renaming[norm_key]] = project.pop(key)
if entry_points:
project["entry-points"] = {
name: [f"{k} = {v}" for k, v in group.items()]
for name, group in entry_points.items()
}
def _copy_command_options(pyproject: dict, dist: "Distribution", filename: _Path):
tool_table = pyproject.get("tool", {})
cmdclass = tool_table.get("setuptools", {}).get("cmdclass", {})
valid_options = _valid_command_options(cmdclass)
cmd_opts = dist.command_options
for cmd, config in pyproject.get("tool", {}).get("distutils", {}).items():
cmd = json_compatible_key(cmd)
valid = valid_options.get(cmd, set())
cmd_opts.setdefault(cmd, {})
for key, value in config.items():
key = json_compatible_key(key)
cmd_opts[cmd][key] = (str(filename), value)
if key not in valid:
# To avoid removing options that are specified dynamically we
# just log a warn...
_logger.warning(f"Command option {cmd}.{key} is not defined")
def _valid_command_options(cmdclass: Mapping = EMPTY) -> Dict[str, Set[str]]:
from .._importlib import metadata
from setuptools.dist import Distribution
valid_options = {"global": _normalise_cmd_options(Distribution.global_options)}
unloaded_entry_points = metadata.entry_points(group='distutils.commands')
loaded_entry_points = (_load_ep(ep) for ep in unloaded_entry_points)
entry_points = (ep for ep in loaded_entry_points if ep)
for cmd, cmd_class in chain(entry_points, cmdclass.items()):
opts = valid_options.get(cmd, set())
opts = opts | _normalise_cmd_options(getattr(cmd_class, "user_options", []))
valid_options[cmd] = opts
return valid_options
def _load_ep(ep: "metadata.EntryPoint") -> Optional[Tuple[str, Type]]:
# Ignore all the errors
try:
return (ep.name, ep.load())
except Exception as ex:
msg = f"{ex.__class__.__name__} while trying to load entry-point {ep.name}"
_logger.warning(f"{msg}: {ex}")
return None
def _normalise_cmd_option_key(name: str) -> str:
return json_compatible_key(name).strip("_=")
def _normalise_cmd_options(desc: List[Tuple[str, Optional[str], str]]) -> Set[str]:
return {_normalise_cmd_option_key(fancy_option[0]) for fancy_option in desc}
def _attrgetter(attr):
"""
Similar to ``operator.attrgetter`` but returns None if ``attr`` is not found
>>> from types import SimpleNamespace
>>> obj = SimpleNamespace(a=42, b=SimpleNamespace(c=13))
>>> _attrgetter("a")(obj)
42
>>> _attrgetter("b.c")(obj)
13
>>> _attrgetter("d")(obj) is None
True
"""
return partial(reduce, lambda acc, x: getattr(acc, x, None), attr.split("."))
def _some_attrgetter(*items):
"""
Return the first "truth-y" attribute or None
>>> from types import SimpleNamespace
>>> obj = SimpleNamespace(a=42, b=SimpleNamespace(c=13))
>>> _some_attrgetter("d", "a", "b.c")(obj)
42
>>> _some_attrgetter("d", "e", "b.c", "a")(obj)
13
>>> _some_attrgetter("d", "e", "f")(obj) is None
True
"""
def _acessor(obj):
values = (_attrgetter(i)(obj) for i in items)
return next((i for i in values if i is not None), None)
return _acessor
PYPROJECT_CORRESPONDENCE: Dict[str, _Correspondence] = {
"readme": _long_description,
"license": _license,
"authors": partial(_people, kind="author"),
"maintainers": partial(_people, kind="maintainer"),
"urls": _project_urls,
"dependencies": _dependencies,
"optional_dependencies": _optional_dependencies,
"requires_python": _python_requires,
}
TOOL_TABLE_RENAMES = {"script_files": "scripts"}
SETUPTOOLS_PATCHES = {"long_description_content_type", "project_urls",
"provides_extras", "license_file", "license_files"}
_PREVIOUSLY_DEFINED = {
"name": _attrgetter("metadata.name"),
"version": _attrgetter("metadata.version"),
"description": _attrgetter("metadata.description"),
"readme": _attrgetter("metadata.long_description"),
"requires-python": _some_attrgetter("python_requires", "metadata.python_requires"),
"license": _attrgetter("metadata.license"),
"authors": _some_attrgetter("metadata.author", "metadata.author_email"),
"maintainers": _some_attrgetter("metadata.maintainer", "metadata.maintainer_email"),
"keywords": _attrgetter("metadata.keywords"),
"classifiers": _attrgetter("metadata.classifiers"),
"urls": _attrgetter("metadata.project_urls"),
"entry-points": _attrgetter("entry_points"),
"dependencies": _some_attrgetter("_orig_install_requires", "install_requires"),
"optional-dependencies": _some_attrgetter("_orig_extras_require", "extras_require"),
}
class _WouldIgnoreField(UserWarning):
"""Inform users that ``pyproject.toml`` would overwrite previously defined metadata:
!!\n\n
##########################################################################
# configuration would be ignored/result in error due to `pyproject.toml` #
##########################################################################
The following seems to be defined outside of `pyproject.toml`:
`{field} = {value!r}`
According to the spec (see the link bellow), however, setuptools CANNOT
consider this value unless {field!r} is listed as `dynamic`.
https://packaging.python.org/en/latest/specifications/declaring-project-metadata/
For the time being, `setuptools` will still consider the given value (as a
**transitional** measure), but please note that future releases of setuptools will
follow strictly the standard.
To prevent this warning, you can list {field!r} under `dynamic` or alternatively
remove the `[project]` table from your file and rely entirely on other means of
configuration.
\n\n!!
"""
@classmethod
def message(cls, field, value):
from inspect import cleandoc
msg = "\n".join(cls.__doc__.splitlines()[1:])
return cleandoc(msg.format(field=field, value=value))
@@ -1,34 +0,0 @@
from functools import reduce
from typing import Any, Callable, Dict
from . import formats
from .error_reporting import detailed_errors, ValidationError
from .extra_validations import EXTRA_VALIDATIONS
from .fastjsonschema_exceptions import JsonSchemaException, JsonSchemaValueException
from .fastjsonschema_validations import validate as _validate
__all__ = [
"validate",
"FORMAT_FUNCTIONS",
"EXTRA_VALIDATIONS",
"ValidationError",
"JsonSchemaException",
"JsonSchemaValueException",
]
FORMAT_FUNCTIONS: Dict[str, Callable[[str], bool]] = {
fn.__name__.replace("_", "-"): fn
for fn in formats.__dict__.values()
if callable(fn) and not fn.__name__.startswith("_")
}
def validate(data: Any) -> bool:
"""Validate the given ``data`` object using JSON Schema
This function raises ``ValidationError`` if ``data`` is invalid.
"""
with detailed_errors():
_validate(data, custom_formats=FORMAT_FUNCTIONS)
reduce(lambda acc, fn: fn(acc), EXTRA_VALIDATIONS, data)
return True
@@ -1,318 +0,0 @@
import io
import json
import logging
import os
import re
from contextlib import contextmanager
from textwrap import indent, wrap
from typing import Any, Dict, Iterator, List, Optional, Sequence, Union, cast
from .fastjsonschema_exceptions import JsonSchemaValueException
_logger = logging.getLogger(__name__)
_MESSAGE_REPLACEMENTS = {
"must be named by propertyName definition": "keys must be named by",
"one of contains definition": "at least one item that matches",
" same as const definition:": "",
"only specified items": "only items matching the definition",
}
_SKIP_DETAILS = (
"must not be empty",
"is always invalid",
"must not be there",
)
_NEED_DETAILS = {"anyOf", "oneOf", "anyOf", "contains", "propertyNames", "not", "items"}
_CAMEL_CASE_SPLITTER = re.compile(r"\W+|([A-Z][^A-Z\W]*)")
_IDENTIFIER = re.compile(r"^[\w_]+$", re.I)
_TOML_JARGON = {
"object": "table",
"property": "key",
"properties": "keys",
"property names": "keys",
}
class ValidationError(JsonSchemaValueException):
"""Report violations of a given JSON schema.
This class extends :exc:`~fastjsonschema.JsonSchemaValueException`
by adding the following properties:
- ``summary``: an improved version of the ``JsonSchemaValueException`` error message
with only the necessary information)
- ``details``: more contextual information about the error like the failing schema
itself and the value that violates the schema.
Depending on the level of the verbosity of the ``logging`` configuration
the exception message will be only ``summary`` (default) or a combination of
``summary`` and ``details`` (when the logging level is set to :obj:`logging.DEBUG`).
"""
summary = ""
details = ""
_original_message = ""
@classmethod
def _from_jsonschema(cls, ex: JsonSchemaValueException):
formatter = _ErrorFormatting(ex)
obj = cls(str(formatter), ex.value, formatter.name, ex.definition, ex.rule)
debug_code = os.getenv("JSONSCHEMA_DEBUG_CODE_GENERATION", "false").lower()
if debug_code != "false": # pragma: no cover
obj.__cause__, obj.__traceback__ = ex.__cause__, ex.__traceback__
obj._original_message = ex.message
obj.summary = formatter.summary
obj.details = formatter.details
return obj
@contextmanager
def detailed_errors():
try:
yield
except JsonSchemaValueException as ex:
raise ValidationError._from_jsonschema(ex) from None
class _ErrorFormatting:
def __init__(self, ex: JsonSchemaValueException):
self.ex = ex
self.name = f"`{self._simplify_name(ex.name)}`"
self._original_message = self.ex.message.replace(ex.name, self.name)
self._summary = ""
self._details = ""
def __str__(self) -> str:
if _logger.getEffectiveLevel() <= logging.DEBUG and self.details:
return f"{self.summary}\n\n{self.details}"
return self.summary
@property
def summary(self) -> str:
if not self._summary:
self._summary = self._expand_summary()
return self._summary
@property
def details(self) -> str:
if not self._details:
self._details = self._expand_details()
return self._details
def _simplify_name(self, name):
x = len("data.")
return name[x:] if name.startswith("data.") else name
def _expand_summary(self):
msg = self._original_message
for bad, repl in _MESSAGE_REPLACEMENTS.items():
msg = msg.replace(bad, repl)
if any(substring in msg for substring in _SKIP_DETAILS):
return msg
schema = self.ex.rule_definition
if self.ex.rule in _NEED_DETAILS and schema:
summary = _SummaryWriter(_TOML_JARGON)
return f"{msg}:\n\n{indent(summary(schema), ' ')}"
return msg
def _expand_details(self) -> str:
optional = []
desc_lines = self.ex.definition.pop("$$description", [])
desc = self.ex.definition.pop("description", None) or " ".join(desc_lines)
if desc:
description = "\n".join(
wrap(
desc,
width=80,
initial_indent=" ",
subsequent_indent=" ",
break_long_words=False,
)
)
optional.append(f"DESCRIPTION:\n{description}")
schema = json.dumps(self.ex.definition, indent=4)
value = json.dumps(self.ex.value, indent=4)
defaults = [
f"GIVEN VALUE:\n{indent(value, ' ')}",
f"OFFENDING RULE: {self.ex.rule!r}",
f"DEFINITION:\n{indent(schema, ' ')}",
]
return "\n\n".join(optional + defaults)
class _SummaryWriter:
_IGNORE = {"description", "default", "title", "examples"}
def __init__(self, jargon: Optional[Dict[str, str]] = None):
self.jargon: Dict[str, str] = jargon or {}
# Clarify confusing terms
self._terms = {
"anyOf": "at least one of the following",
"oneOf": "exactly one of the following",
"allOf": "all of the following",
"not": "(*NOT* the following)",
"prefixItems": f"{self._jargon('items')} (in order)",
"items": "items",
"contains": "contains at least one of",
"propertyNames": (
f"non-predefined acceptable {self._jargon('property names')}"
),
"patternProperties": f"{self._jargon('properties')} named via pattern",
"const": "predefined value",
"enum": "one of",
}
# Attributes that indicate that the definition is easy and can be done
# inline (e.g. string and number)
self._guess_inline_defs = [
"enum",
"const",
"maxLength",
"minLength",
"pattern",
"format",
"minimum",
"maximum",
"exclusiveMinimum",
"exclusiveMaximum",
"multipleOf",
]
def _jargon(self, term: Union[str, List[str]]) -> Union[str, List[str]]:
if isinstance(term, list):
return [self.jargon.get(t, t) for t in term]
return self.jargon.get(term, term)
def __call__(
self,
schema: Union[dict, List[dict]],
prefix: str = "",
*,
_path: Sequence[str] = (),
) -> str:
if isinstance(schema, list):
return self._handle_list(schema, prefix, _path)
filtered = self._filter_unecessary(schema, _path)
simple = self._handle_simple_dict(filtered, _path)
if simple:
return f"{prefix}{simple}"
child_prefix = self._child_prefix(prefix, " ")
item_prefix = self._child_prefix(prefix, "- ")
indent = len(prefix) * " "
with io.StringIO() as buffer:
for i, (key, value) in enumerate(filtered.items()):
child_path = [*_path, key]
line_prefix = prefix if i == 0 else indent
buffer.write(f"{line_prefix}{self._label(child_path)}:")
# ^ just the first item should receive the complete prefix
if isinstance(value, dict):
filtered = self._filter_unecessary(value, child_path)
simple = self._handle_simple_dict(filtered, child_path)
buffer.write(
f" {simple}"
if simple
else f"\n{self(value, child_prefix, _path=child_path)}"
)
elif isinstance(value, list) and (
key != "type" or self._is_property(child_path)
):
children = self._handle_list(value, item_prefix, child_path)
sep = " " if children.startswith("[") else "\n"
buffer.write(f"{sep}{children}")
else:
buffer.write(f" {self._value(value, child_path)}\n")
return buffer.getvalue()
def _is_unecessary(self, path: Sequence[str]) -> bool:
if self._is_property(path) or not path: # empty path => instruction @ root
return False
key = path[-1]
return any(key.startswith(k) for k in "$_") or key in self._IGNORE
def _filter_unecessary(self, schema: dict, path: Sequence[str]):
return {
key: value
for key, value in schema.items()
if not self._is_unecessary([*path, key])
}
def _handle_simple_dict(self, value: dict, path: Sequence[str]) -> Optional[str]:
inline = any(p in value for p in self._guess_inline_defs)
simple = not any(isinstance(v, (list, dict)) for v in value.values())
if inline or simple:
return f"{{{', '.join(self._inline_attrs(value, path))}}}\n"
return None
def _handle_list(
self, schemas: list, prefix: str = "", path: Sequence[str] = ()
) -> str:
if self._is_unecessary(path):
return ""
repr_ = repr(schemas)
if all(not isinstance(e, (dict, list)) for e in schemas) and len(repr_) < 60:
return f"{repr_}\n"
item_prefix = self._child_prefix(prefix, "- ")
return "".join(
self(v, item_prefix, _path=[*path, f"[{i}]"]) for i, v in enumerate(schemas)
)
def _is_property(self, path: Sequence[str]):
"""Check if the given path can correspond to an arbitrarily named property"""
counter = 0
for key in path[-2::-1]:
if key not in {"properties", "patternProperties"}:
break
counter += 1
# If the counter if even, the path correspond to a JSON Schema keyword
# otherwise it can be any arbitrary string naming a property
return counter % 2 == 1
def _label(self, path: Sequence[str]) -> str:
*parents, key = path
if not self._is_property(path):
norm_key = _separate_terms(key)
return self._terms.get(key) or " ".join(self._jargon(norm_key))
if parents[-1] == "patternProperties":
return f"(regex {key!r})"
return repr(key) # property name
def _value(self, value: Any, path: Sequence[str]) -> str:
if path[-1] == "type" and not self._is_property(path):
type_ = self._jargon(value)
return (
f"[{', '.join(type_)}]" if isinstance(value, list) else cast(str, type_)
)
return repr(value)
def _inline_attrs(self, schema: dict, path: Sequence[str]) -> Iterator[str]:
for key, value in schema.items():
child_path = [*path, key]
yield f"{self._label(child_path)}: {self._value(value, child_path)}"
def _child_prefix(self, parent_prefix: str, child_prefix: str) -> str:
return len(parent_prefix) * " " + child_prefix
def _separate_terms(word: str) -> List[str]:
"""
>>> _separate_terms("FooBar-foo")
['foo', 'bar', 'foo']
"""
return [w.lower() for w in _CAMEL_CASE_SPLITTER.split(word) if w]
@@ -1,36 +0,0 @@
"""The purpose of this module is implement PEP 621 validations that are
difficult to express as a JSON Schema (or that are not supported by the current
JSON Schema library).
"""
from typing import Mapping, TypeVar
from .fastjsonschema_exceptions import JsonSchemaValueException
T = TypeVar("T", bound=Mapping)
class RedefiningStaticFieldAsDynamic(JsonSchemaValueException):
"""According to PEP 621:
Build back-ends MUST raise an error if the metadata specifies a field
statically as well as being listed in dynamic.
"""
def validate_project_dynamic(pyproject: T) -> T:
project_table = pyproject.get("project", {})
dynamic = project_table.get("dynamic", [])
for field in dynamic:
if field in project_table:
msg = f"You cannot provide a value for `project.{field}` and "
msg += "list it under `project.dynamic` at the same time"
name = f"data.project.{field}"
value = {field: project_table[field], "...": " # ...", "dynamic": dynamic}
raise RedefiningStaticFieldAsDynamic(msg, value, name, rule="PEP 621")
return pyproject
EXTRA_VALIDATIONS = (validate_project_dynamic,)
@@ -1,51 +0,0 @@
import re
SPLIT_RE = re.compile(r'[\.\[\]]+')
class JsonSchemaException(ValueError):
"""
Base exception of ``fastjsonschema`` library.
"""
class JsonSchemaValueException(JsonSchemaException):
"""
Exception raised by validation function. Available properties:
* ``message`` containing human-readable information what is wrong (e.g. ``data.property[index] must be smaller than or equal to 42``),
* invalid ``value`` (e.g. ``60``),
* ``name`` of a path in the data structure (e.g. ``data.property[index]``),
* ``path`` as an array in the data structure (e.g. ``['data', 'property', 'index']``),
* the whole ``definition`` which the ``value`` has to fulfil (e.g. ``{'type': 'number', 'maximum': 42}``),
* ``rule`` which the ``value`` is breaking (e.g. ``maximum``)
* and ``rule_definition`` (e.g. ``42``).
.. versionchanged:: 2.14.0
Added all extra properties.
"""
def __init__(self, message, value=None, name=None, definition=None, rule=None):
super().__init__(message)
self.message = message
self.value = value
self.name = name
self.definition = definition
self.rule = rule
@property
def path(self):
return [item for item in SPLIT_RE.split(self.name) if item != '']
@property
def rule_definition(self):
if not self.rule or not self.definition:
return None
return self.definition.get(self.rule)
class JsonSchemaDefinitionException(JsonSchemaException):
"""
Exception raised by generator of validation function.
"""
File diff suppressed because one or more lines are too long
@@ -1,257 +0,0 @@
import logging
import os
import re
import string
import typing
from itertools import chain as _chain
_logger = logging.getLogger(__name__)
# -------------------------------------------------------------------------------------
# PEP 440
VERSION_PATTERN = r"""
v?
(?:
(?:(?P<epoch>[0-9]+)!)? # epoch
(?P<release>[0-9]+(?:\.[0-9]+)*) # release segment
(?P<pre> # pre-release
[-_\.]?
(?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview))
[-_\.]?
(?P<pre_n>[0-9]+)?
)?
(?P<post> # post release
(?:-(?P<post_n1>[0-9]+))
|
(?:
[-_\.]?
(?P<post_l>post|rev|r)
[-_\.]?
(?P<post_n2>[0-9]+)?
)
)?
(?P<dev> # dev release
[-_\.]?
(?P<dev_l>dev)
[-_\.]?
(?P<dev_n>[0-9]+)?
)?
)
(?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))? # local version
"""
VERSION_REGEX = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.X | re.I)
def pep440(version: str) -> bool:
return VERSION_REGEX.match(version) is not None
# -------------------------------------------------------------------------------------
# PEP 508
PEP508_IDENTIFIER_PATTERN = r"([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])"
PEP508_IDENTIFIER_REGEX = re.compile(f"^{PEP508_IDENTIFIER_PATTERN}$", re.I)
def pep508_identifier(name: str) -> bool:
return PEP508_IDENTIFIER_REGEX.match(name) is not None
try:
try:
from packaging import requirements as _req
except ImportError: # pragma: no cover
# let's try setuptools vendored version
from setuptools._vendor.packaging import requirements as _req # type: ignore
def pep508(value: str) -> bool:
try:
_req.Requirement(value)
return True
except _req.InvalidRequirement:
return False
except ImportError: # pragma: no cover
_logger.warning(
"Could not find an installation of `packaging`. Requirements, dependencies and "
"versions might not be validated. "
"To enforce validation, please install `packaging`."
)
def pep508(value: str) -> bool:
return True
def pep508_versionspec(value: str) -> bool:
"""Expression that can be used to specify/lock versions (including ranges)"""
if any(c in value for c in (";", "]", "@")):
# In PEP 508:
# conditional markers, extras and URL specs are not included in the
# versionspec
return False
# Let's pretend we have a dependency called `requirement` with the given
# version spec, then we can re-use the pep508 function for validation:
return pep508(f"requirement{value}")
# -------------------------------------------------------------------------------------
# PEP 517
def pep517_backend_reference(value: str) -> bool:
module, _, obj = value.partition(":")
identifiers = (i.strip() for i in _chain(module.split("."), obj.split(".")))
return all(python_identifier(i) for i in identifiers if i)
# -------------------------------------------------------------------------------------
# Classifiers - PEP 301
def _download_classifiers() -> str:
import cgi
from urllib.request import urlopen
url = "https://pypi.org/pypi?:action=list_classifiers"
with urlopen(url) as response:
content_type = response.getheader("content-type", "text/plain")
encoding = cgi.parse_header(content_type)[1].get("charset", "utf-8")
return response.read().decode(encoding)
class _TroveClassifier:
"""The ``trove_classifiers`` package is the official way of validating classifiers,
however this package might not be always available.
As a workaround we can still download a list from PyPI.
We also don't want to be over strict about it, so simply skipping silently is an
option (classifiers will be validated anyway during the upload to PyPI).
"""
def __init__(self):
self.downloaded: typing.Union[None, False, typing.Set[str]] = None
self._skip_download = False
# None => not cached yet
# False => cache not available
self.__name__ = "trove_classifier" # Emulate a public function
def _disable_download(self):
# This is a private API. Only setuptools has the consent of using it.
self._skip_download = True
def __call__(self, value: str) -> bool:
if self.downloaded is False or self._skip_download is True:
return True
if os.getenv("NO_NETWORK") or os.getenv("VALIDATE_PYPROJECT_NO_NETWORK"):
self.downloaded = False
msg = (
"Install ``trove-classifiers`` to ensure proper validation. "
"Skipping download of classifiers list from PyPI (NO_NETWORK)."
)
_logger.debug(msg)
return True
if self.downloaded is None:
msg = (
"Install ``trove-classifiers`` to ensure proper validation. "
"Meanwhile a list of classifiers will be downloaded from PyPI."
)
_logger.debug(msg)
try:
self.downloaded = set(_download_classifiers().splitlines())
except Exception:
self.downloaded = False
_logger.debug("Problem with download, skipping validation")
return True
return value in self.downloaded or value.lower().startswith("private ::")
try:
from trove_classifiers import classifiers as _trove_classifiers
def trove_classifier(value: str) -> bool:
return value in _trove_classifiers or value.lower().startswith("private ::")
except ImportError: # pragma: no cover
trove_classifier = _TroveClassifier()
# -------------------------------------------------------------------------------------
# Non-PEP related
def url(value: str) -> bool:
from urllib.parse import urlparse
try:
parts = urlparse(value)
if not parts.scheme:
_logger.warning(
"For maximum compatibility please make sure to include a "
"`scheme` prefix in your URL (e.g. 'http://'). "
f"Given value: {value}"
)
if not (value.startswith("/") or value.startswith("\\") or "@" in value):
parts = urlparse(f"http://{value}")
return bool(parts.scheme and parts.netloc)
except Exception:
return False
# https://packaging.python.org/specifications/entry-points/
ENTRYPOINT_PATTERN = r"[^\[\s=]([^=]*[^\s=])?"
ENTRYPOINT_REGEX = re.compile(f"^{ENTRYPOINT_PATTERN}$", re.I)
RECOMMEDED_ENTRYPOINT_PATTERN = r"[\w.-]+"
RECOMMEDED_ENTRYPOINT_REGEX = re.compile(f"^{RECOMMEDED_ENTRYPOINT_PATTERN}$", re.I)
ENTRYPOINT_GROUP_PATTERN = r"\w+(\.\w+)*"
ENTRYPOINT_GROUP_REGEX = re.compile(f"^{ENTRYPOINT_GROUP_PATTERN}$", re.I)
def python_identifier(value: str) -> bool:
return value.isidentifier()
def python_qualified_identifier(value: str) -> bool:
if value.startswith(".") or value.endswith("."):
return False
return all(python_identifier(m) for m in value.split("."))
def python_module_name(value: str) -> bool:
return python_qualified_identifier(value)
def python_entrypoint_group(value: str) -> bool:
return ENTRYPOINT_GROUP_REGEX.match(value) is not None
def python_entrypoint_name(value: str) -> bool:
if not ENTRYPOINT_REGEX.match(value):
return False
if not RECOMMEDED_ENTRYPOINT_REGEX.match(value):
msg = f"Entry point `{value}` does not follow recommended pattern: "
msg += RECOMMEDED_ENTRYPOINT_PATTERN
_logger.warning(msg)
return True
def python_entrypoint_reference(value: str) -> bool:
module, _, rest = value.partition(":")
if "[" in rest:
obj, _, extras_ = rest.partition("[")
if extras_.strip()[-1] != "]":
return False
extras = (x.strip() for x in extras_.strip(string.whitespace + "[]").split(","))
if not all(pep508_identifier(e) for e in extras):
return False
_logger.warning(f"`{value}` - using extras for entry points is not recommended")
else:
obj = rest
module_parts = module.split(".")
identifiers = _chain(module_parts, obj.split(".")) if rest else module_parts
return all(python_identifier(i.strip()) for i in identifiers)
@@ -1,481 +0,0 @@
"""Utility functions to expand configuration directives or special values
(such glob patterns).
We can split the process of interpreting configuration files into 2 steps:
1. The parsing the file contents from strings to value objects
that can be understand by Python (for example a string with a comma
separated list of keywords into an actual Python list of strings).
2. The expansion (or post-processing) of these values according to the
semantics ``setuptools`` assign to them (for example a configuration field
with the ``file:`` directive should be expanded from a list of file paths to
a single string with the contents of those files concatenated)
This module focus on the second step, and therefore allow sharing the expansion
functions among several configuration file formats.
"""
import ast
import importlib
import io
import os
import sys
import warnings
from glob import iglob
from configparser import ConfigParser
from importlib.machinery import ModuleSpec
from itertools import chain
from typing import (
TYPE_CHECKING,
Callable,
Dict,
Iterable,
Iterator,
List,
Mapping,
Optional,
Tuple,
TypeVar,
Union,
cast
)
from types import ModuleType
from distutils.errors import DistutilsOptionError
if TYPE_CHECKING:
from setuptools.dist import Distribution # noqa
from setuptools.discovery import ConfigDiscovery # noqa
from distutils.dist import DistributionMetadata # noqa
chain_iter = chain.from_iterable
_Path = Union[str, os.PathLike]
_K = TypeVar("_K")
_V = TypeVar("_V", covariant=True)
class StaticModule:
"""Proxy to a module object that avoids executing arbitrary code."""
def __init__(self, name: str, spec: ModuleSpec):
with open(spec.origin) as strm: # type: ignore
src = strm.read()
module = ast.parse(src)
vars(self).update(locals())
del self.self
def __getattr__(self, attr):
"""Attempt to load an attribute "statically", via :func:`ast.literal_eval`."""
try:
assignment_expressions = (
statement
for statement in self.module.body
if isinstance(statement, ast.Assign)
)
expressions_with_target = (
(statement, target)
for statement in assignment_expressions
for target in statement.targets
)
matching_values = (
statement.value
for statement, target in expressions_with_target
if isinstance(target, ast.Name) and target.id == attr
)
return next(ast.literal_eval(value) for value in matching_values)
except Exception as e:
raise AttributeError(f"{self.name} has no attribute {attr}") from e
def glob_relative(
patterns: Iterable[str], root_dir: Optional[_Path] = None
) -> List[str]:
"""Expand the list of glob patterns, but preserving relative paths.
:param list[str] patterns: List of glob patterns
:param str root_dir: Path to which globs should be relative
(current directory by default)
:rtype: list
"""
glob_characters = {'*', '?', '[', ']', '{', '}'}
expanded_values = []
root_dir = root_dir or os.getcwd()
for value in patterns:
# Has globby characters?
if any(char in value for char in glob_characters):
# then expand the glob pattern while keeping paths *relative*:
glob_path = os.path.abspath(os.path.join(root_dir, value))
expanded_values.extend(sorted(
os.path.relpath(path, root_dir).replace(os.sep, "/")
for path in iglob(glob_path, recursive=True)))
else:
# take the value as-is
path = os.path.relpath(value, root_dir).replace(os.sep, "/")
expanded_values.append(path)
return expanded_values
def read_files(filepaths: Union[str, bytes, Iterable[_Path]], root_dir=None) -> str:
"""Return the content of the files concatenated using ``\n`` as str
This function is sandboxed and won't reach anything outside ``root_dir``
(By default ``root_dir`` is the current directory).
"""
from setuptools.extern.more_itertools import always_iterable
root_dir = os.path.abspath(root_dir or os.getcwd())
_filepaths = (os.path.join(root_dir, path) for path in always_iterable(filepaths))
return '\n'.join(
_read_file(path)
for path in _filter_existing_files(_filepaths)
if _assert_local(path, root_dir)
)
def _filter_existing_files(filepaths: Iterable[_Path]) -> Iterator[_Path]:
for path in filepaths:
if os.path.isfile(path):
yield path
else:
warnings.warn(f"File {path!r} cannot be found")
def _read_file(filepath: Union[bytes, _Path]) -> str:
with io.open(filepath, encoding='utf-8') as f:
return f.read()
def _assert_local(filepath: _Path, root_dir: str):
if not os.path.abspath(filepath).startswith(root_dir):
msg = f"Cannot access {filepath!r} (or anything outside {root_dir!r})"
raise DistutilsOptionError(msg)
return True
def read_attr(
attr_desc: str,
package_dir: Optional[Mapping[str, str]] = None,
root_dir: Optional[_Path] = None
):
"""Reads the value of an attribute from a module.
This function will try to read the attributed statically first
(via :func:`ast.literal_eval`), and only evaluate the module if it fails.
Examples:
read_attr("package.attr")
read_attr("package.module.attr")
:param str attr_desc: Dot-separated string describing how to reach the
attribute (see examples above)
:param dict[str, str] package_dir: Mapping of package names to their
location in disk (represented by paths relative to ``root_dir``).
:param str root_dir: Path to directory containing all the packages in
``package_dir`` (current directory by default).
:rtype: str
"""
root_dir = root_dir or os.getcwd()
attrs_path = attr_desc.strip().split('.')
attr_name = attrs_path.pop()
module_name = '.'.join(attrs_path)
module_name = module_name or '__init__'
_parent_path, path, module_name = _find_module(module_name, package_dir, root_dir)
spec = _find_spec(module_name, path)
try:
return getattr(StaticModule(module_name, spec), attr_name)
except Exception:
# fallback to evaluate module
module = _load_spec(spec, module_name)
return getattr(module, attr_name)
def _find_spec(module_name: str, module_path: Optional[_Path]) -> ModuleSpec:
spec = importlib.util.spec_from_file_location(module_name, module_path)
spec = spec or importlib.util.find_spec(module_name)
if spec is None:
raise ModuleNotFoundError(module_name)
return spec
def _load_spec(spec: ModuleSpec, module_name: str) -> ModuleType:
name = getattr(spec, "__name__", module_name)
if name in sys.modules:
return sys.modules[name]
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module # cache (it also ensures `==` works on loaded items)
spec.loader.exec_module(module) # type: ignore
return module
def _find_module(
module_name: str, package_dir: Optional[Mapping[str, str]], root_dir: _Path
) -> Tuple[_Path, Optional[str], str]:
"""Given a module (that could normally be imported by ``module_name``
after the build is complete), find the path to the parent directory where
it is contained and the canonical name that could be used to import it
considering the ``package_dir`` in the build configuration and ``root_dir``
"""
parent_path = root_dir
module_parts = module_name.split('.')
if package_dir:
if module_parts[0] in package_dir:
# A custom path was specified for the module we want to import
custom_path = package_dir[module_parts[0]]
parts = custom_path.rsplit('/', 1)
if len(parts) > 1:
parent_path = os.path.join(root_dir, parts[0])
parent_module = parts[1]
else:
parent_module = custom_path
module_name = ".".join([parent_module, *module_parts[1:]])
elif '' in package_dir:
# A custom parent directory was specified for all root modules
parent_path = os.path.join(root_dir, package_dir[''])
path_start = os.path.join(parent_path, *module_name.split("."))
candidates = chain(
(f"{path_start}.py", os.path.join(path_start, "__init__.py")),
iglob(f"{path_start}.*")
)
module_path = next((x for x in candidates if os.path.isfile(x)), None)
return parent_path, module_path, module_name
def resolve_class(
qualified_class_name: str,
package_dir: Optional[Mapping[str, str]] = None,
root_dir: Optional[_Path] = None
) -> Callable:
"""Given a qualified class name, return the associated class object"""
root_dir = root_dir or os.getcwd()
idx = qualified_class_name.rfind('.')
class_name = qualified_class_name[idx + 1 :]
pkg_name = qualified_class_name[:idx]
_parent_path, path, module_name = _find_module(pkg_name, package_dir, root_dir)
module = _load_spec(_find_spec(module_name, path), module_name)
return getattr(module, class_name)
def cmdclass(
values: Dict[str, str],
package_dir: Optional[Mapping[str, str]] = None,
root_dir: Optional[_Path] = None
) -> Dict[str, Callable]:
"""Given a dictionary mapping command names to strings for qualified class
names, apply :func:`resolve_class` to the dict values.
"""
return {k: resolve_class(v, package_dir, root_dir) for k, v in values.items()}
def find_packages(
*,
namespaces=True,
fill_package_dir: Optional[Dict[str, str]] = None,
root_dir: Optional[_Path] = None,
**kwargs
) -> List[str]:
"""Works similarly to :func:`setuptools.find_packages`, but with all
arguments given as keyword arguments. Moreover, ``where`` can be given
as a list (the results will be simply concatenated).
When the additional keyword argument ``namespaces`` is ``True``, it will
behave like :func:`setuptools.find_namespace_packages`` (i.e. include
implicit namespaces as per :pep:`420`).
The ``where`` argument will be considered relative to ``root_dir`` (or the current
working directory when ``root_dir`` is not given).
If the ``fill_package_dir`` argument is passed, this function will consider it as a
similar data structure to the ``package_dir`` configuration parameter add fill-in
any missing package location.
:rtype: list
"""
from setuptools.discovery import construct_package_dir
from setuptools.extern.more_itertools import unique_everseen, always_iterable
if namespaces:
from setuptools.discovery import PEP420PackageFinder as PackageFinder
else:
from setuptools.discovery import PackageFinder # type: ignore
root_dir = root_dir or os.curdir
where = kwargs.pop('where', ['.'])
packages: List[str] = []
fill_package_dir = {} if fill_package_dir is None else fill_package_dir
search = list(unique_everseen(always_iterable(where)))
if len(search) == 1 and all(not _same_path(search[0], x) for x in (".", root_dir)):
fill_package_dir.setdefault("", search[0])
for path in search:
package_path = _nest_path(root_dir, path)
pkgs = PackageFinder.find(package_path, **kwargs)
packages.extend(pkgs)
if pkgs and not (
fill_package_dir.get("") == path
or os.path.samefile(package_path, root_dir)
):
fill_package_dir.update(construct_package_dir(pkgs, path))
return packages
def _same_path(p1: _Path, p2: _Path) -> bool:
"""Differs from os.path.samefile because it does not require paths to exist.
Purely string based (no comparison between i-nodes).
>>> _same_path("a/b", "./a/b")
True
>>> _same_path("a/b", "a/./b")
True
>>> _same_path("a/b", "././a/b")
True
>>> _same_path("a/b", "./a/b/c/..")
True
>>> _same_path("a/b", "../a/b/c")
False
>>> _same_path("a", "a/b")
False
"""
return os.path.normpath(p1) == os.path.normpath(p2)
def _nest_path(parent: _Path, path: _Path) -> str:
path = parent if path in {".", ""} else os.path.join(parent, path)
return os.path.normpath(path)
def version(value: Union[Callable, Iterable[Union[str, int]], str]) -> str:
"""When getting the version directly from an attribute,
it should be normalised to string.
"""
if callable(value):
value = value()
value = cast(Iterable[Union[str, int]], value)
if not isinstance(value, str):
if hasattr(value, '__iter__'):
value = '.'.join(map(str, value))
else:
value = '%s' % value
return value
def canonic_package_data(package_data: dict) -> dict:
if "*" in package_data:
package_data[""] = package_data.pop("*")
return package_data
def canonic_data_files(
data_files: Union[list, dict], root_dir: Optional[_Path] = None
) -> List[Tuple[str, List[str]]]:
"""For compatibility with ``setup.py``, ``data_files`` should be a list
of pairs instead of a dict.
This function also expands glob patterns.
"""
if isinstance(data_files, list):
return data_files
return [
(dest, glob_relative(patterns, root_dir))
for dest, patterns in data_files.items()
]
def entry_points(text: str, text_source="entry-points") -> Dict[str, dict]:
"""Given the contents of entry-points file,
process it into a 2-level dictionary (``dict[str, dict[str, str]]``).
The first level keys are entry-point groups, the second level keys are
entry-point names, and the second level values are references to objects
(that correspond to the entry-point value).
"""
parser = ConfigParser(default_section=None, delimiters=("=",)) # type: ignore
parser.optionxform = str # case sensitive
parser.read_string(text, text_source)
groups = {k: dict(v.items()) for k, v in parser.items()}
groups.pop(parser.default_section, None)
return groups
class EnsurePackagesDiscovered:
"""Some expand functions require all the packages to already be discovered before
they run, e.g. :func:`read_attr`, :func:`resolve_class`, :func:`cmdclass`.
Therefore in some cases we will need to run autodiscovery during the evaluation of
the configuration. However, it is better to postpone calling package discovery as
much as possible, because some parameters can influence it (e.g. ``package_dir``),
and those might not have been processed yet.
"""
def __init__(self, distribution: "Distribution"):
self._dist = distribution
self._called = False
def __call__(self):
"""Trigger the automatic package discovery, if it is still necessary."""
if not self._called:
self._called = True
self._dist.set_defaults(name=False) # Skip name, we can still be parsing
def __enter__(self):
return self
def __exit__(self, _exc_type, _exc_value, _traceback):
if self._called:
self._dist.set_defaults.analyse_name() # Now we can set a default name
def _get_package_dir(self) -> Mapping[str, str]:
self()
pkg_dir = self._dist.package_dir
return {} if pkg_dir is None else pkg_dir
@property
def package_dir(self) -> Mapping[str, str]:
"""Proxy to ``package_dir`` that may trigger auto-discovery when used."""
return LazyMappingProxy(self._get_package_dir)
class LazyMappingProxy(Mapping[_K, _V]):
"""Mapping proxy that delays resolving the target object, until really needed.
>>> def obtain_mapping():
... print("Running expensive function!")
... return {"key": "value", "other key": "other value"}
>>> mapping = LazyMappingProxy(obtain_mapping)
>>> mapping["key"]
Running expensive function!
'value'
>>> mapping["other key"]
'other value'
"""
def __init__(self, obtain_mapping_value: Callable[[], Mapping[_K, _V]]):
self._obtain = obtain_mapping_value
self._value: Optional[Mapping[_K, _V]] = None
def _target(self) -> Mapping[_K, _V]:
if self._value is None:
self._value = self._obtain()
return self._value
def __getitem__(self, key: _K) -> _V:
return self._target()[key]
def __len__(self) -> int:
return len(self._target())
def __iter__(self) -> Iterator[_K]:
return iter(self._target())
@@ -1,440 +0,0 @@
"""Load setuptools configuration from ``pyproject.toml`` files"""
import logging
import os
import warnings
from contextlib import contextmanager
from functools import partial
from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Union
from setuptools.errors import FileError, OptionError
from . import expand as _expand
from ._apply_pyprojecttoml import apply as _apply
from ._apply_pyprojecttoml import _PREVIOUSLY_DEFINED, _WouldIgnoreField
if TYPE_CHECKING:
from setuptools.dist import Distribution # noqa
_Path = Union[str, os.PathLike]
_logger = logging.getLogger(__name__)
def load_file(filepath: _Path) -> dict:
from setuptools.extern import tomli # type: ignore
with open(filepath, "rb") as file:
return tomli.load(file)
def validate(config: dict, filepath: _Path) -> bool:
from . import _validate_pyproject as validator
trove_classifier = validator.FORMAT_FUNCTIONS.get("trove-classifier")
if hasattr(trove_classifier, "_disable_download"):
# Improve reproducibility by default. See issue 31 for validate-pyproject.
trove_classifier._disable_download() # type: ignore
try:
return validator.validate(config)
except validator.ValidationError as ex:
_logger.error(f"configuration error: {ex.summary}") # type: ignore
_logger.debug(ex.details) # type: ignore
error = ValueError(f"invalid pyproject.toml config: {ex.name}") # type: ignore
raise error from None
def apply_configuration(
dist: "Distribution",
filepath: _Path,
ignore_option_errors=False,
) -> "Distribution":
"""Apply the configuration from a ``pyproject.toml`` file into an existing
distribution object.
"""
config = read_configuration(filepath, True, ignore_option_errors, dist)
return _apply(dist, config, filepath)
def read_configuration(
filepath: _Path,
expand=True,
ignore_option_errors=False,
dist: Optional["Distribution"] = None,
):
"""Read given configuration file and returns options from it as a dict.
:param str|unicode filepath: Path to configuration file in the ``pyproject.toml``
format.
:param bool expand: Whether to expand directives and other computed values
(i.e. post-process the given configuration)
:param bool ignore_option_errors: Whether to silently ignore
options, values of which could not be resolved (e.g. due to exceptions
in directives such as file:, attr:, etc.).
If False exceptions are propagated as expected.
:param Distribution|None: Distribution object to which the configuration refers.
If not given a dummy object will be created and discarded after the
configuration is read. This is used for auto-discovery of packages in the case
a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.
When ``expand=False`` this object is simply ignored.
:rtype: dict
"""
filepath = os.path.abspath(filepath)
if not os.path.isfile(filepath):
raise FileError(f"Configuration file {filepath!r} does not exist.")
asdict = load_file(filepath) or {}
project_table = asdict.get("project", {})
tool_table = asdict.get("tool", {})
setuptools_table = tool_table.get("setuptools", {})
if not asdict or not (project_table or setuptools_table):
return {} # User is not using pyproject to configure setuptools
# TODO: Remove the following once the feature stabilizes:
msg = (
"Support for project metadata in `pyproject.toml` is still experimental "
"and may be removed (or change) in future releases."
)
warnings.warn(msg, _ExperimentalProjectMetadata)
# There is an overall sense in the community that making include_package_data=True
# the default would be an improvement.
# `ini2toml` backfills include_package_data=False when nothing is explicitly given,
# therefore setting a default here is backwards compatible.
orig_setuptools_table = setuptools_table.copy()
if dist and getattr(dist, "include_package_data") is not None:
setuptools_table.setdefault("include-package-data", dist.include_package_data)
else:
setuptools_table.setdefault("include-package-data", True)
# Persist changes:
asdict["tool"] = tool_table
tool_table["setuptools"] = setuptools_table
try:
# Don't complain about unrelated errors (e.g. tools not using the "tool" table)
subset = {"project": project_table, "tool": {"setuptools": setuptools_table}}
validate(subset, filepath)
except Exception as ex:
# TODO: Remove the following once the feature stabilizes:
if _skip_bad_config(project_table, orig_setuptools_table, dist):
return {}
# TODO: After the previous statement is removed the try/except can be replaced
# by the _ignore_errors context manager.
if ignore_option_errors:
_logger.debug(f"ignored error: {ex.__class__.__name__} - {ex}")
else:
raise # re-raise exception
if expand:
root_dir = os.path.dirname(filepath)
return expand_configuration(asdict, root_dir, ignore_option_errors, dist)
return asdict
def _skip_bad_config(
project_cfg: dict, setuptools_cfg: dict, dist: Optional["Distribution"]
) -> bool:
"""Be temporarily forgiving with invalid ``pyproject.toml``"""
# See pypa/setuptools#3199 and pypa/cibuildwheel#1064
if dist is None or (
dist.metadata.name is None
and dist.metadata.version is None
and dist.install_requires is None
):
# It seems that the build is not getting any configuration from other places
return False
if setuptools_cfg:
# If `[tool.setuptools]` is set, then `pyproject.toml` config is intentional
return False
given_config = set(project_cfg.keys())
popular_subset = {"name", "version", "python_requires", "requires-python"}
if given_config <= popular_subset:
# It seems that the docs in cibuildtool has been inadvertently encouraging users
# to create `pyproject.toml` files that are not compliant with the standards.
# Let's be forgiving for the time being.
warnings.warn(_InvalidFile.message(), _InvalidFile, stacklevel=2)
return True
return False
def expand_configuration(
config: dict,
root_dir: Optional[_Path] = None,
ignore_option_errors: bool = False,
dist: Optional["Distribution"] = None,
) -> dict:
"""Given a configuration with unresolved fields (e.g. dynamic, cmdclass, ...)
find their final values.
:param dict config: Dict containing the configuration for the distribution
:param str root_dir: Top-level directory for the distribution/project
(the same directory where ``pyproject.toml`` is place)
:param bool ignore_option_errors: see :func:`read_configuration`
:param Distribution|None: Distribution object to which the configuration refers.
If not given a dummy object will be created and discarded after the
configuration is read. Used in the case a dynamic configuration
(e.g. ``attr`` or ``cmdclass``).
:rtype: dict
"""
return _ConfigExpander(config, root_dir, ignore_option_errors, dist).expand()
class _ConfigExpander:
def __init__(
self,
config: dict,
root_dir: Optional[_Path] = None,
ignore_option_errors: bool = False,
dist: Optional["Distribution"] = None,
):
self.config = config
self.root_dir = root_dir or os.getcwd()
self.project_cfg = config.get("project", {})
self.dynamic = self.project_cfg.get("dynamic", [])
self.setuptools_cfg = config.get("tool", {}).get("setuptools", {})
self.dynamic_cfg = self.setuptools_cfg.get("dynamic", {})
self.ignore_option_errors = ignore_option_errors
self._dist = dist
def _ensure_dist(self) -> "Distribution":
from setuptools.dist import Distribution
attrs = {"src_root": self.root_dir, "name": self.project_cfg.get("name", None)}
return self._dist or Distribution(attrs)
def _process_field(self, container: dict, field: str, fn: Callable):
if field in container:
with _ignore_errors(self.ignore_option_errors):
container[field] = fn(container[field])
def _canonic_package_data(self, field="package-data"):
package_data = self.setuptools_cfg.get(field, {})
return _expand.canonic_package_data(package_data)
def expand(self):
self._expand_packages()
self._canonic_package_data()
self._canonic_package_data("exclude-package-data")
# A distribution object is required for discovering the correct package_dir
dist = self._ensure_dist()
with _EnsurePackagesDiscovered(dist, self.setuptools_cfg) as ensure_discovered:
package_dir = ensure_discovered.package_dir
self._expand_data_files()
self._expand_cmdclass(package_dir)
self._expand_all_dynamic(dist, package_dir)
return self.config
def _expand_packages(self):
packages = self.setuptools_cfg.get("packages")
if packages is None or isinstance(packages, (list, tuple)):
return
find = packages.get("find")
if isinstance(find, dict):
find["root_dir"] = self.root_dir
find["fill_package_dir"] = self.setuptools_cfg.setdefault("package-dir", {})
with _ignore_errors(self.ignore_option_errors):
self.setuptools_cfg["packages"] = _expand.find_packages(**find)
def _expand_data_files(self):
data_files = partial(_expand.canonic_data_files, root_dir=self.root_dir)
self._process_field(self.setuptools_cfg, "data-files", data_files)
def _expand_cmdclass(self, package_dir: Mapping[str, str]):
root_dir = self.root_dir
cmdclass = partial(_expand.cmdclass, package_dir=package_dir, root_dir=root_dir)
self._process_field(self.setuptools_cfg, "cmdclass", cmdclass)
def _expand_all_dynamic(self, dist: "Distribution", package_dir: Mapping[str, str]):
special = ( # need special handling
"version",
"readme",
"entry-points",
"scripts",
"gui-scripts",
"classifiers",
)
# `_obtain` functions are assumed to raise appropriate exceptions/warnings.
obtained_dynamic = {
field: self._obtain(dist, field, package_dir)
for field in self.dynamic
if field not in special
}
obtained_dynamic.update(
self._obtain_entry_points(dist, package_dir) or {},
version=self._obtain_version(dist, package_dir),
readme=self._obtain_readme(dist),
classifiers=self._obtain_classifiers(dist),
)
# `None` indicates there is nothing in `tool.setuptools.dynamic` but the value
# might have already been set by setup.py/extensions, so avoid overwriting.
updates = {k: v for k, v in obtained_dynamic.items() if v is not None}
self.project_cfg.update(updates)
def _ensure_previously_set(self, dist: "Distribution", field: str):
previous = _PREVIOUSLY_DEFINED[field](dist)
if previous is None and not self.ignore_option_errors:
msg = (
f"No configuration found for dynamic {field!r}.\n"
"Some dynamic fields need to be specified via `tool.setuptools.dynamic`"
"\nothers must be specified via the equivalent attribute in `setup.py`."
)
raise OptionError(msg)
def _obtain(self, dist: "Distribution", field: str, package_dir: Mapping[str, str]):
if field in self.dynamic_cfg:
directive = self.dynamic_cfg[field]
with _ignore_errors(self.ignore_option_errors):
root_dir = self.root_dir
if "file" in directive:
return _expand.read_files(directive["file"], root_dir)
if "attr" in directive:
return _expand.read_attr(directive["attr"], package_dir, root_dir)
msg = f"invalid `tool.setuptools.dynamic.{field}`: {directive!r}"
raise ValueError(msg)
return None
self._ensure_previously_set(dist, field)
return None
def _obtain_version(self, dist: "Distribution", package_dir: Mapping[str, str]):
# Since plugins can set version, let's silently skip if it cannot be obtained
if "version" in self.dynamic and "version" in self.dynamic_cfg:
return _expand.version(self._obtain(dist, "version", package_dir))
return None
def _obtain_readme(self, dist: "Distribution") -> Optional[Dict[str, str]]:
if "readme" not in self.dynamic:
return None
dynamic_cfg = self.dynamic_cfg
if "readme" in dynamic_cfg:
return {
"text": self._obtain(dist, "readme", {}),
"content-type": dynamic_cfg["readme"].get("content-type", "text/x-rst"),
}
self._ensure_previously_set(dist, "readme")
return None
def _obtain_entry_points(
self, dist: "Distribution", package_dir: Mapping[str, str]
) -> Optional[Dict[str, dict]]:
fields = ("entry-points", "scripts", "gui-scripts")
if not any(field in self.dynamic for field in fields):
return None
text = self._obtain(dist, "entry-points", package_dir)
if text is None:
return None
groups = _expand.entry_points(text)
expanded = {"entry-points": groups}
def _set_scripts(field: str, group: str):
if group in groups:
value = groups.pop(group)
if field not in self.dynamic:
msg = _WouldIgnoreField.message(field, value)
warnings.warn(msg, _WouldIgnoreField)
# TODO: Don't set field when support for pyproject.toml stabilizes
# instead raise an error as specified in PEP 621
expanded[field] = value
_set_scripts("scripts", "console_scripts")
_set_scripts("gui-scripts", "gui_scripts")
return expanded
def _obtain_classifiers(self, dist: "Distribution"):
if "classifiers" in self.dynamic:
value = self._obtain(dist, "classifiers", {})
if value:
return value.splitlines()
return None
@contextmanager
def _ignore_errors(ignore_option_errors: bool):
if not ignore_option_errors:
yield
return
try:
yield
except Exception as ex:
_logger.debug(f"ignored error: {ex.__class__.__name__} - {ex}")
class _EnsurePackagesDiscovered(_expand.EnsurePackagesDiscovered):
def __init__(self, distribution: "Distribution", setuptools_cfg: dict):
super().__init__(distribution)
self._setuptools_cfg = setuptools_cfg
def __enter__(self):
"""When entering the context, the values of ``packages``, ``py_modules`` and
``package_dir`` that are missing in ``dist`` are copied from ``setuptools_cfg``.
"""
dist, cfg = self._dist, self._setuptools_cfg
package_dir: Dict[str, str] = cfg.setdefault("package-dir", {})
package_dir.update(dist.package_dir or {})
dist.package_dir = package_dir # needs to be the same object
dist.set_defaults._ignore_ext_modules() # pyproject.toml-specific behaviour
# Set `py_modules` and `packages` in dist to short-circuit auto-discovery,
# but avoid overwriting empty lists purposefully set by users.
if dist.py_modules is None:
dist.py_modules = cfg.get("py-modules")
if dist.packages is None:
dist.packages = cfg.get("packages")
return super().__enter__()
def __exit__(self, exc_type, exc_value, traceback):
"""When exiting the context, if values of ``packages``, ``py_modules`` and
``package_dir`` are missing in ``setuptools_cfg``, copy from ``dist``.
"""
# If anything was discovered set them back, so they count in the final config.
self._setuptools_cfg.setdefault("packages", self._dist.packages)
self._setuptools_cfg.setdefault("py-modules", self._dist.py_modules)
return super().__exit__(exc_type, exc_value, traceback)
class _ExperimentalProjectMetadata(UserWarning):
"""Explicitly inform users that `pyproject.toml` configuration is experimental"""
class _InvalidFile(UserWarning):
"""Inform users that the given `pyproject.toml` is experimental:
!!\n\n
############################
# Invalid `pyproject.toml` #
############################
Any configurations in `pyproject.toml` will be ignored.
Please note that future releases of setuptools will halt the build process
if an invalid file is given.
To prevent setuptools from considering `pyproject.toml` please
DO NOT include the `[project]` or `[tool.setuptools]` tables in your file.
\n\n!!
"""
@classmethod
def message(cls):
from inspect import cleandoc
msg = "\n".join(cls.__doc__.splitlines()[1:])
return cleandoc(msg)
@@ -1,689 +0,0 @@
"""Load setuptools configuration from ``setup.cfg`` files"""
import os
import warnings
import functools
from collections import defaultdict
from functools import partial
from functools import wraps
from typing import (TYPE_CHECKING, Callable, Any, Dict, Generic, Iterable, List,
Optional, Tuple, TypeVar, Union)
from distutils.errors import DistutilsOptionError, DistutilsFileError
from setuptools.extern.packaging.version import Version, InvalidVersion
from setuptools.extern.packaging.specifiers import SpecifierSet
from . import expand
if TYPE_CHECKING:
from setuptools.dist import Distribution # noqa
from distutils.dist import DistributionMetadata # noqa
_Path = Union[str, os.PathLike]
SingleCommandOptions = Dict["str", Tuple["str", Any]]
"""Dict that associate the name of the options of a particular command to a
tuple. The first element of the tuple indicates the origin of the option value
(e.g. the name of the configuration file where it was read from),
while the second element of the tuple is the option value itself
"""
AllCommandOptions = Dict["str", SingleCommandOptions] # cmd name => its options
Target = TypeVar("Target", bound=Union["Distribution", "DistributionMetadata"])
def read_configuration(
filepath: _Path,
find_others=False,
ignore_option_errors=False
) -> dict:
"""Read given configuration file and returns options from it as a dict.
:param str|unicode filepath: Path to configuration file
to get options from.
:param bool find_others: Whether to search for other configuration files
which could be on in various places.
:param bool ignore_option_errors: Whether to silently ignore
options, values of which could not be resolved (e.g. due to exceptions
in directives such as file:, attr:, etc.).
If False exceptions are propagated as expected.
:rtype: dict
"""
from setuptools.dist import Distribution
dist = Distribution()
filenames = dist.find_config_files() if find_others else []
handlers = _apply(dist, filepath, filenames, ignore_option_errors)
return configuration_to_dict(handlers)
def apply_configuration(dist: "Distribution", filepath: _Path) -> "Distribution":
"""Apply the configuration from a ``setup.cfg`` file into an existing
distribution object.
"""
_apply(dist, filepath)
dist._finalize_requires()
return dist
def _apply(
dist: "Distribution", filepath: _Path,
other_files: Iterable[_Path] = (),
ignore_option_errors: bool = False,
) -> Tuple["ConfigHandler", ...]:
"""Read configuration from ``filepath`` and applies to the ``dist`` object."""
from setuptools.dist import _Distribution
filepath = os.path.abspath(filepath)
if not os.path.isfile(filepath):
raise DistutilsFileError('Configuration file %s does not exist.' % filepath)
current_directory = os.getcwd()
os.chdir(os.path.dirname(filepath))
filenames = [*other_files, filepath]
try:
_Distribution.parse_config_files(dist, filenames=filenames)
handlers = parse_configuration(
dist, dist.command_options, ignore_option_errors=ignore_option_errors
)
dist._finalize_license_files()
finally:
os.chdir(current_directory)
return handlers
def _get_option(target_obj: Target, key: str):
"""
Given a target object and option key, get that option from
the target object, either through a get_{key} method or
from an attribute directly.
"""
getter_name = 'get_{key}'.format(**locals())
by_attribute = functools.partial(getattr, target_obj, key)
getter = getattr(target_obj, getter_name, by_attribute)
return getter()
def configuration_to_dict(handlers: Tuple["ConfigHandler", ...]) -> dict:
"""Returns configuration data gathered by given handlers as a dict.
:param list[ConfigHandler] handlers: Handlers list,
usually from parse_configuration()
:rtype: dict
"""
config_dict: dict = defaultdict(dict)
for handler in handlers:
for option in handler.set_options:
value = _get_option(handler.target_obj, option)
config_dict[handler.section_prefix][option] = value
return config_dict
def parse_configuration(
distribution: "Distribution",
command_options: AllCommandOptions,
ignore_option_errors=False
) -> Tuple["ConfigMetadataHandler", "ConfigOptionsHandler"]:
"""Performs additional parsing of configuration options
for a distribution.
Returns a list of used option handlers.
:param Distribution distribution:
:param dict command_options:
:param bool ignore_option_errors: Whether to silently ignore
options, values of which could not be resolved (e.g. due to exceptions
in directives such as file:, attr:, etc.).
If False exceptions are propagated as expected.
:rtype: list
"""
with expand.EnsurePackagesDiscovered(distribution) as ensure_discovered:
options = ConfigOptionsHandler(
distribution,
command_options,
ignore_option_errors,
ensure_discovered,
)
options.parse()
if not distribution.package_dir:
distribution.package_dir = options.package_dir # Filled by `find_packages`
meta = ConfigMetadataHandler(
distribution.metadata,
command_options,
ignore_option_errors,
ensure_discovered,
distribution.package_dir,
distribution.src_root,
)
meta.parse()
return meta, options
class ConfigHandler(Generic[Target]):
"""Handles metadata supplied in configuration files."""
section_prefix: str
"""Prefix for config sections handled by this handler.
Must be provided by class heirs.
"""
aliases: Dict[str, str] = {}
"""Options aliases.
For compatibility with various packages. E.g.: d2to1 and pbr.
Note: `-` in keys is replaced with `_` by config parser.
"""
def __init__(
self,
target_obj: Target,
options: AllCommandOptions,
ignore_option_errors,
ensure_discovered: expand.EnsurePackagesDiscovered,
):
sections: AllCommandOptions = {}
section_prefix = self.section_prefix
for section_name, section_options in options.items():
if not section_name.startswith(section_prefix):
continue
section_name = section_name.replace(section_prefix, '').strip('.')
sections[section_name] = section_options
self.ignore_option_errors = ignore_option_errors
self.target_obj = target_obj
self.sections = sections
self.set_options: List[str] = []
self.ensure_discovered = ensure_discovered
@property
def parsers(self):
"""Metadata item name to parser function mapping."""
raise NotImplementedError(
'%s must provide .parsers property' % self.__class__.__name__
)
def __setitem__(self, option_name, value):
unknown = tuple()
target_obj = self.target_obj
# Translate alias into real name.
option_name = self.aliases.get(option_name, option_name)
current_value = getattr(target_obj, option_name, unknown)
if current_value is unknown:
raise KeyError(option_name)
if current_value:
# Already inhabited. Skipping.
return
skip_option = False
parser = self.parsers.get(option_name)
if parser:
try:
value = parser(value)
except Exception:
skip_option = True
if not self.ignore_option_errors:
raise
if skip_option:
return
setter = getattr(target_obj, 'set_%s' % option_name, None)
if setter is None:
setattr(target_obj, option_name, value)
else:
setter(value)
self.set_options.append(option_name)
@classmethod
def _parse_list(cls, value, separator=','):
"""Represents value as a list.
Value is split either by separator (defaults to comma) or by lines.
:param value:
:param separator: List items separator character.
:rtype: list
"""
if isinstance(value, list): # _get_parser_compound case
return value
if '\n' in value:
value = value.splitlines()
else:
value = value.split(separator)
return [chunk.strip() for chunk in value if chunk.strip()]
@classmethod
def _parse_dict(cls, value):
"""Represents value as a dict.
:param value:
:rtype: dict
"""
separator = '='
result = {}
for line in cls._parse_list(value):
key, sep, val = line.partition(separator)
if sep != separator:
raise DistutilsOptionError(
'Unable to parse option value to dict: %s' % value
)
result[key.strip()] = val.strip()
return result
@classmethod
def _parse_bool(cls, value):
"""Represents value as boolean.
:param value:
:rtype: bool
"""
value = value.lower()
return value in ('1', 'true', 'yes')
@classmethod
def _exclude_files_parser(cls, key):
"""Returns a parser function to make sure field inputs
are not files.
Parses a value after getting the key so error messages are
more informative.
:param key:
:rtype: callable
"""
def parser(value):
exclude_directive = 'file:'
if value.startswith(exclude_directive):
raise ValueError(
'Only strings are accepted for the {0} field, '
'files are not accepted'.format(key)
)
return value
return parser
@classmethod
def _parse_file(cls, value, root_dir: _Path):
"""Represents value as a string, allowing including text
from nearest files using `file:` directive.
Directive is sandboxed and won't reach anything outside
directory with setup.py.
Examples:
file: README.rst, CHANGELOG.md, src/file.txt
:param str value:
:rtype: str
"""
include_directive = 'file:'
if not isinstance(value, str):
return value
if not value.startswith(include_directive):
return value
spec = value[len(include_directive) :]
filepaths = (path.strip() for path in spec.split(','))
return expand.read_files(filepaths, root_dir)
def _parse_attr(self, value, package_dir, root_dir: _Path):
"""Represents value as a module attribute.
Examples:
attr: package.attr
attr: package.module.attr
:param str value:
:rtype: str
"""
attr_directive = 'attr:'
if not value.startswith(attr_directive):
return value
attr_desc = value.replace(attr_directive, '')
# Make sure package_dir is populated correctly, so `attr:` directives can work
package_dir.update(self.ensure_discovered.package_dir)
return expand.read_attr(attr_desc, package_dir, root_dir)
@classmethod
def _get_parser_compound(cls, *parse_methods):
"""Returns parser function to represents value as a list.
Parses a value applying given methods one after another.
:param parse_methods:
:rtype: callable
"""
def parse(value):
parsed = value
for method in parse_methods:
parsed = method(parsed)
return parsed
return parse
@classmethod
def _parse_section_to_dict(cls, section_options, values_parser=None):
"""Parses section options into a dictionary.
Optionally applies a given parser to values.
:param dict section_options:
:param callable values_parser:
:rtype: dict
"""
value = {}
values_parser = values_parser or (lambda val: val)
for key, (_, val) in section_options.items():
value[key] = values_parser(val)
return value
def parse_section(self, section_options):
"""Parses configuration file section.
:param dict section_options:
"""
for (name, (_, value)) in section_options.items():
try:
self[name] = value
except KeyError:
pass # Keep silent for a new option may appear anytime.
def parse(self):
"""Parses configuration file items from one
or more related sections.
"""
for section_name, section_options in self.sections.items():
method_postfix = ''
if section_name: # [section.option] variant
method_postfix = '_%s' % section_name
section_parser_method: Optional[Callable] = getattr(
self,
# Dots in section names are translated into dunderscores.
('parse_section%s' % method_postfix).replace('.', '__'),
None,
)
if section_parser_method is None:
raise DistutilsOptionError(
'Unsupported distribution option section: [%s.%s]'
% (self.section_prefix, section_name)
)
section_parser_method(section_options)
def _deprecated_config_handler(self, func, msg, warning_class):
"""this function will wrap around parameters that are deprecated
:param msg: deprecation message
:param warning_class: class of warning exception to be raised
:param func: function to be wrapped around
"""
@wraps(func)
def config_handler(*args, **kwargs):
warnings.warn(msg, warning_class)
return func(*args, **kwargs)
return config_handler
class ConfigMetadataHandler(ConfigHandler["DistributionMetadata"]):
section_prefix = 'metadata'
aliases = {
'home_page': 'url',
'summary': 'description',
'classifier': 'classifiers',
'platform': 'platforms',
}
strict_mode = False
"""We need to keep it loose, to be partially compatible with
`pbr` and `d2to1` packages which also uses `metadata` section.
"""
def __init__(
self,
target_obj: "DistributionMetadata",
options: AllCommandOptions,
ignore_option_errors: bool,
ensure_discovered: expand.EnsurePackagesDiscovered,
package_dir: Optional[dict] = None,
root_dir: _Path = os.curdir
):
super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)
self.package_dir = package_dir
self.root_dir = root_dir
@property
def parsers(self):
"""Metadata item name to parser function mapping."""
parse_list = self._parse_list
parse_file = partial(self._parse_file, root_dir=self.root_dir)
parse_dict = self._parse_dict
exclude_files_parser = self._exclude_files_parser
return {
'platforms': parse_list,
'keywords': parse_list,
'provides': parse_list,
'requires': self._deprecated_config_handler(
parse_list,
"The requires parameter is deprecated, please use "
"install_requires for runtime dependencies.",
DeprecationWarning,
),
'obsoletes': parse_list,
'classifiers': self._get_parser_compound(parse_file, parse_list),
'license': exclude_files_parser('license'),
'license_file': self._deprecated_config_handler(
exclude_files_parser('license_file'),
"The license_file parameter is deprecated, "
"use license_files instead.",
DeprecationWarning,
),
'license_files': parse_list,
'description': parse_file,
'long_description': parse_file,
'version': self._parse_version,
'project_urls': parse_dict,
}
def _parse_version(self, value):
"""Parses `version` option value.
:param value:
:rtype: str
"""
version = self._parse_file(value, self.root_dir)
if version != value:
version = version.strip()
# Be strict about versions loaded from file because it's easy to
# accidentally include newlines and other unintended content
try:
Version(version)
except InvalidVersion:
tmpl = (
'Version loaded from {value} does not '
'comply with PEP 440: {version}'
)
raise DistutilsOptionError(tmpl.format(**locals()))
return version
return expand.version(self._parse_attr(value, self.package_dir, self.root_dir))
class ConfigOptionsHandler(ConfigHandler["Distribution"]):
section_prefix = 'options'
def __init__(
self,
target_obj: "Distribution",
options: AllCommandOptions,
ignore_option_errors: bool,
ensure_discovered: expand.EnsurePackagesDiscovered,
):
super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)
self.root_dir = target_obj.src_root
self.package_dir: Dict[str, str] = {} # To be filled by `find_packages`
@property
def parsers(self):
"""Metadata item name to parser function mapping."""
parse_list = self._parse_list
parse_list_semicolon = partial(self._parse_list, separator=';')
parse_bool = self._parse_bool
parse_dict = self._parse_dict
parse_cmdclass = self._parse_cmdclass
parse_file = partial(self._parse_file, root_dir=self.root_dir)
return {
'zip_safe': parse_bool,
'include_package_data': parse_bool,
'package_dir': parse_dict,
'scripts': parse_list,
'eager_resources': parse_list,
'dependency_links': parse_list,
'namespace_packages': parse_list,
'install_requires': parse_list_semicolon,
'setup_requires': parse_list_semicolon,
'tests_require': parse_list_semicolon,
'packages': self._parse_packages,
'entry_points': parse_file,
'py_modules': parse_list,
'python_requires': SpecifierSet,
'cmdclass': parse_cmdclass,
}
def _parse_cmdclass(self, value):
package_dir = self.ensure_discovered.package_dir
return expand.cmdclass(self._parse_dict(value), package_dir, self.root_dir)
def _parse_packages(self, value):
"""Parses `packages` option value.
:param value:
:rtype: list
"""
find_directives = ['find:', 'find_namespace:']
trimmed_value = value.strip()
if trimmed_value not in find_directives:
return self._parse_list(value)
# Read function arguments from a dedicated section.
find_kwargs = self.parse_section_packages__find(
self.sections.get('packages.find', {})
)
find_kwargs.update(
namespaces=(trimmed_value == find_directives[1]),
root_dir=self.root_dir,
fill_package_dir=self.package_dir,
)
return expand.find_packages(**find_kwargs)
def parse_section_packages__find(self, section_options):
"""Parses `packages.find` configuration file section.
To be used in conjunction with _parse_packages().
:param dict section_options:
"""
section_data = self._parse_section_to_dict(section_options, self._parse_list)
valid_keys = ['where', 'include', 'exclude']
find_kwargs = dict(
[(k, v) for k, v in section_data.items() if k in valid_keys and v]
)
where = find_kwargs.get('where')
if where is not None:
find_kwargs['where'] = where[0] # cast list to single val
return find_kwargs
def parse_section_entry_points(self, section_options):
"""Parses `entry_points` configuration file section.
:param dict section_options:
"""
parsed = self._parse_section_to_dict(section_options, self._parse_list)
self['entry_points'] = parsed
def _parse_package_data(self, section_options):
package_data = self._parse_section_to_dict(section_options, self._parse_list)
return expand.canonic_package_data(package_data)
def parse_section_package_data(self, section_options):
"""Parses `package_data` configuration file section.
:param dict section_options:
"""
self['package_data'] = self._parse_package_data(section_options)
def parse_section_exclude_package_data(self, section_options):
"""Parses `exclude_package_data` configuration file section.
:param dict section_options:
"""
self['exclude_package_data'] = self._parse_package_data(section_options)
def parse_section_extras_require(self, section_options):
"""Parses `extras_require` configuration file section.
:param dict section_options:
"""
parse_list = partial(self._parse_list, separator=';')
parsed = self._parse_section_to_dict(section_options, parse_list)
self['extras_require'] = parsed
def parse_section_data_files(self, section_options):
"""Parses `data_files` configuration file section.
:param dict section_options:
"""
parsed = self._parse_section_to_dict(section_options, self._parse_list)
self['data_files'] = expand.canonic_data_files(parsed, self.root_dir)
@@ -1,588 +0,0 @@
"""Automatic discovery of Python modules and packages (for inclusion in the
distribution) and other config values.
For the purposes of this module, the following nomenclature is used:
- "src-layout": a directory representing a Python project that contains a "src"
folder. Everything under the "src" folder is meant to be included in the
distribution when packaging the project. Example::
.
tox.ini
pyproject.toml
src/
mypkg/
__init__.py
mymodule.py
my_data_file.txt
- "flat-layout": a Python project that does not use "src-layout" but instead
have a directory under the project root for each package::
.
tox.ini
pyproject.toml
mypkg/
__init__.py
mymodule.py
my_data_file.txt
- "single-module": a project that contains a single Python script direct under
the project root (no directory used)::
.
tox.ini
pyproject.toml
mymodule.py
"""
import itertools
import os
from fnmatch import fnmatchcase
from glob import glob
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Callable, Dict, Iterator, Iterable, List, Optional, Tuple, Union
import _distutils_hack.override # noqa: F401
from distutils import log
from distutils.util import convert_path
_Path = Union[str, os.PathLike]
_Filter = Callable[[str], bool]
StrIter = Iterator[str]
chain_iter = itertools.chain.from_iterable
if TYPE_CHECKING:
from setuptools import Distribution # noqa
def _valid_name(path: _Path) -> bool:
# Ignore invalid names that cannot be imported directly
return os.path.basename(path).isidentifier()
class _Finder:
"""Base class that exposes functionality for module/package finders"""
ALWAYS_EXCLUDE: Tuple[str, ...] = ()
DEFAULT_EXCLUDE: Tuple[str, ...] = ()
@classmethod
def find(
cls,
where: _Path = '.',
exclude: Iterable[str] = (),
include: Iterable[str] = ('*',)
) -> List[str]:
"""Return a list of all Python items (packages or modules, depending on
the finder implementation) found within directory 'where'.
'where' is the root directory which will be searched.
It should be supplied as a "cross-platform" (i.e. URL-style) path;
it will be converted to the appropriate local path syntax.
'exclude' is a sequence of names to exclude; '*' can be used
as a wildcard in the names.
When finding packages, 'foo.*' will exclude all subpackages of 'foo'
(but not 'foo' itself).
'include' is a sequence of names to include.
If it's specified, only the named items will be included.
If it's not specified, all found items will be included.
'include' can contain shell style wildcard patterns just like
'exclude'.
"""
exclude = exclude or cls.DEFAULT_EXCLUDE
return list(
cls._find_iter(
convert_path(str(where)),
cls._build_filter(*cls.ALWAYS_EXCLUDE, *exclude),
cls._build_filter(*include),
)
)
@classmethod
def _find_iter(cls, where: _Path, exclude: _Filter, include: _Filter) -> StrIter:
raise NotImplementedError
@staticmethod
def _build_filter(*patterns: str) -> _Filter:
"""
Given a list of patterns, return a callable that will be true only if
the input matches at least one of the patterns.
"""
return lambda name: any(fnmatchcase(name, pat) for pat in patterns)
class PackageFinder(_Finder):
"""
Generate a list of all Python packages found within a directory
"""
ALWAYS_EXCLUDE = ("ez_setup", "*__pycache__")
@classmethod
def _find_iter(cls, where: _Path, exclude: _Filter, include: _Filter) -> StrIter:
"""
All the packages found in 'where' that pass the 'include' filter, but
not the 'exclude' filter.
"""
for root, dirs, files in os.walk(str(where), followlinks=True):
# Copy dirs to iterate over it, then empty dirs.
all_dirs = dirs[:]
dirs[:] = []
for dir in all_dirs:
full_path = os.path.join(root, dir)
rel_path = os.path.relpath(full_path, where)
package = rel_path.replace(os.path.sep, '.')
# Skip directory trees that are not valid packages
if '.' in dir or not cls._looks_like_package(full_path, package):
continue
# Should this package be included?
if include(package) and not exclude(package):
yield package
# Keep searching subdirectories, as there may be more packages
# down there, even if the parent was excluded.
dirs.append(dir)
@staticmethod
def _looks_like_package(path: _Path, _package_name: str) -> bool:
"""Does a directory look like a package?"""
return os.path.isfile(os.path.join(path, '__init__.py'))
class PEP420PackageFinder(PackageFinder):
@staticmethod
def _looks_like_package(_path: _Path, _package_name: str) -> bool:
return True
class ModuleFinder(_Finder):
"""Find isolated Python modules.
This function will **not** recurse subdirectories.
"""
@classmethod
def _find_iter(cls, where: _Path, exclude: _Filter, include: _Filter) -> StrIter:
for file in glob(os.path.join(where, "*.py")):
module, _ext = os.path.splitext(os.path.basename(file))
if not cls._looks_like_module(module):
continue
if include(module) and not exclude(module):
yield module
_looks_like_module = staticmethod(_valid_name)
# We have to be extra careful in the case of flat layout to not include files
# and directories not meant for distribution (e.g. tool-related)
class FlatLayoutPackageFinder(PEP420PackageFinder):
_EXCLUDE = (
"ci",
"bin",
"doc",
"docs",
"documentation",
"manpages",
"news",
"changelog",
"test",
"tests",
"unit_test",
"unit_tests",
"example",
"examples",
"scripts",
"tools",
"util",
"utils",
"python",
"build",
"dist",
"venv",
"env",
"requirements",
# ---- Task runners / Build tools ----
"tasks", # invoke
"fabfile", # fabric
"site_scons", # SCons
# ---- Other tools ----
"benchmark",
"benchmarks",
"exercise",
"exercises",
# ---- Hidden directories/Private packages ----
"[._]*",
)
DEFAULT_EXCLUDE = tuple(chain_iter((p, f"{p}.*") for p in _EXCLUDE))
"""Reserved package names"""
@staticmethod
def _looks_like_package(_path: _Path, package_name: str) -> bool:
names = package_name.split('.')
# Consider PEP 561
root_pkg_is_valid = names[0].isidentifier() or names[0].endswith("-stubs")
return root_pkg_is_valid and all(name.isidentifier() for name in names[1:])
class FlatLayoutModuleFinder(ModuleFinder):
DEFAULT_EXCLUDE = (
"setup",
"conftest",
"test",
"tests",
"example",
"examples",
"build",
# ---- Task runners ----
"toxfile",
"noxfile",
"pavement",
"dodo",
"tasks",
"fabfile",
# ---- Other tools ----
"[Ss][Cc]onstruct", # SCons
"conanfile", # Connan: C/C++ build tool
"manage", # Django
"benchmark",
"benchmarks",
"exercise",
"exercises",
# ---- Hidden files/Private modules ----
"[._]*",
)
"""Reserved top-level module names"""
def _find_packages_within(root_pkg: str, pkg_dir: _Path) -> List[str]:
nested = PEP420PackageFinder.find(pkg_dir)
return [root_pkg] + [".".join((root_pkg, n)) for n in nested]
class ConfigDiscovery:
"""Fill-in metadata and options that can be automatically derived
(from other metadata/options, the file system or conventions)
"""
def __init__(self, distribution: "Distribution"):
self.dist = distribution
self._called = False
self._disabled = False
self._skip_ext_modules = False
def _disable(self):
"""Internal API to disable automatic discovery"""
self._disabled = True
def _ignore_ext_modules(self):
"""Internal API to disregard ext_modules.
Normally auto-discovery would not be triggered if ``ext_modules`` are set
(this is done for backward compatibility with existing packages relying on
``setup.py`` or ``setup.cfg``). However, ``setuptools`` can call this function
to ignore given ``ext_modules`` and proceed with the auto-discovery if
``packages`` and ``py_modules`` are not given (e.g. when using pyproject.toml
metadata).
"""
self._skip_ext_modules = True
@property
def _root_dir(self) -> _Path:
# The best is to wait until `src_root` is set in dist, before using _root_dir.
return self.dist.src_root or os.curdir
@property
def _package_dir(self) -> Dict[str, str]:
if self.dist.package_dir is None:
return {}
return self.dist.package_dir
def __call__(self, force=False, name=True, ignore_ext_modules=False):
"""Automatically discover missing configuration fields
and modifies the given ``distribution`` object in-place.
Note that by default this will only have an effect the first time the
``ConfigDiscovery`` object is called.
To repeatedly invoke automatic discovery (e.g. when the project
directory changes), please use ``force=True`` (or create a new
``ConfigDiscovery`` instance).
"""
if force is False and (self._called or self._disabled):
# Avoid overhead of multiple calls
return
self._analyse_package_layout(ignore_ext_modules)
if name:
self.analyse_name() # depends on ``packages`` and ``py_modules``
self._called = True
def _explicitly_specified(self, ignore_ext_modules: bool) -> bool:
"""``True`` if the user has specified some form of package/module listing"""
ignore_ext_modules = ignore_ext_modules or self._skip_ext_modules
ext_modules = not (self.dist.ext_modules is None or ignore_ext_modules)
return (
self.dist.packages is not None
or self.dist.py_modules is not None
or ext_modules
or hasattr(self.dist, "configuration") and self.dist.configuration
# ^ Some projects use numpy.distutils.misc_util.Configuration
)
def _analyse_package_layout(self, ignore_ext_modules: bool) -> bool:
if self._explicitly_specified(ignore_ext_modules):
# For backward compatibility, just try to find modules/packages
# when nothing is given
return True
log.debug(
"No `packages` or `py_modules` configuration, performing "
"automatic discovery."
)
return (
self._analyse_explicit_layout()
or self._analyse_src_layout()
# flat-layout is the trickiest for discovery so it should be last
or self._analyse_flat_layout()
)
def _analyse_explicit_layout(self) -> bool:
"""The user can explicitly give a package layout via ``package_dir``"""
package_dir = self._package_dir.copy() # don't modify directly
package_dir.pop("", None) # This falls under the "src-layout" umbrella
root_dir = self._root_dir
if not package_dir:
return False
log.debug(f"`explicit-layout` detected -- analysing {package_dir}")
pkgs = chain_iter(
_find_packages_within(pkg, os.path.join(root_dir, parent_dir))
for pkg, parent_dir in package_dir.items()
)
self.dist.packages = list(pkgs)
log.debug(f"discovered packages -- {self.dist.packages}")
return True
def _analyse_src_layout(self) -> bool:
"""Try to find all packages or modules under the ``src`` directory
(or anything pointed by ``package_dir[""]``).
The "src-layout" is relatively safe for automatic discovery.
We assume that everything within is meant to be included in the
distribution.
If ``package_dir[""]`` is not given, but the ``src`` directory exists,
this function will set ``package_dir[""] = "src"``.
"""
package_dir = self._package_dir
src_dir = os.path.join(self._root_dir, package_dir.get("", "src"))
if not os.path.isdir(src_dir):
return False
log.debug(f"`src-layout` detected -- analysing {src_dir}")
package_dir.setdefault("", os.path.basename(src_dir))
self.dist.package_dir = package_dir # persist eventual modifications
self.dist.packages = PEP420PackageFinder.find(src_dir)
self.dist.py_modules = ModuleFinder.find(src_dir)
log.debug(f"discovered packages -- {self.dist.packages}")
log.debug(f"discovered py_modules -- {self.dist.py_modules}")
return True
def _analyse_flat_layout(self) -> bool:
"""Try to find all packages and modules under the project root.
Since the ``flat-layout`` is more dangerous in terms of accidentally including
extra files/directories, this function is more conservative and will raise an
error if multiple packages or modules are found.
This assumes that multi-package dists are uncommon and refuse to support that
use case in order to be able to prevent unintended errors.
"""
log.debug(f"`flat-layout` detected -- analysing {self._root_dir}")
return self._analyse_flat_packages() or self._analyse_flat_modules()
def _analyse_flat_packages(self) -> bool:
self.dist.packages = FlatLayoutPackageFinder.find(self._root_dir)
top_level = remove_nested_packages(remove_stubs(self.dist.packages))
log.debug(f"discovered packages -- {self.dist.packages}")
self._ensure_no_accidental_inclusion(top_level, "packages")
return bool(top_level)
def _analyse_flat_modules(self) -> bool:
self.dist.py_modules = FlatLayoutModuleFinder.find(self._root_dir)
log.debug(f"discovered py_modules -- {self.dist.py_modules}")
self._ensure_no_accidental_inclusion(self.dist.py_modules, "modules")
return bool(self.dist.py_modules)
def _ensure_no_accidental_inclusion(self, detected: List[str], kind: str):
if len(detected) > 1:
from inspect import cleandoc
from setuptools.errors import PackageDiscoveryError
msg = f"""Multiple top-level {kind} discovered in a flat-layout: {detected}.
To avoid accidental inclusion of unwanted files or directories,
setuptools will not proceed with this build.
If you are trying to create a single distribution with multiple {kind}
on purpose, you should not rely on automatic discovery.
Instead, consider the following options:
1. set up custom discovery (`find` directive with `include` or `exclude`)
2. use a `src-layout`
3. explicitly set `py_modules` or `packages` with a list of names
To find more information, look for "package discovery" on setuptools docs.
"""
raise PackageDiscoveryError(cleandoc(msg))
def analyse_name(self):
"""The packages/modules are the essential contribution of the author.
Therefore the name of the distribution can be derived from them.
"""
if self.dist.metadata.name or self.dist.name:
# get_name() is not reliable (can return "UNKNOWN")
return None
log.debug("No `name` configuration, performing automatic discovery")
name = (
self._find_name_single_package_or_module()
or self._find_name_from_packages()
)
if name:
self.dist.metadata.name = name
self.dist.name = name
def _find_name_single_package_or_module(self) -> Optional[str]:
"""Exactly one module or package"""
for field in ('packages', 'py_modules'):
items = getattr(self.dist, field, None) or []
if items and len(items) == 1:
log.debug(f"Single module/package detected, name: {items[0]}")
return items[0]
return None
def _find_name_from_packages(self) -> Optional[str]:
"""Try to find the root package that is not a PEP 420 namespace"""
if not self.dist.packages:
return None
packages = remove_stubs(sorted(self.dist.packages, key=len))
package_dir = self.dist.package_dir or {}
parent_pkg = find_parent_package(packages, package_dir, self._root_dir)
if parent_pkg:
log.debug(f"Common parent package detected, name: {parent_pkg}")
return parent_pkg
log.warn("No parent package detected, impossible to derive `name`")
return None
def remove_nested_packages(packages: List[str]) -> List[str]:
"""Remove nested packages from a list of packages.
>>> remove_nested_packages(["a", "a.b1", "a.b2", "a.b1.c1"])
['a']
>>> remove_nested_packages(["a", "b", "c.d", "c.d.e.f", "g.h", "a.a1"])
['a', 'b', 'c.d', 'g.h']
"""
pkgs = sorted(packages, key=len)
top_level = pkgs[:]
size = len(pkgs)
for i, name in enumerate(reversed(pkgs)):
if any(name.startswith(f"{other}.") for other in top_level):
top_level.pop(size - i - 1)
return top_level
def remove_stubs(packages: List[str]) -> List[str]:
"""Remove type stubs (:pep:`561`) from a list of packages.
>>> remove_stubs(["a", "a.b", "a-stubs", "a-stubs.b.c", "b", "c-stubs"])
['a', 'a.b', 'b']
"""
return [pkg for pkg in packages if not pkg.split(".")[0].endswith("-stubs")]
def find_parent_package(
packages: List[str], package_dir: Dict[str, str], root_dir: _Path
) -> Optional[str]:
"""Find the parent package that is not a namespace."""
packages = sorted(packages, key=len)
common_ancestors = []
for i, name in enumerate(packages):
if not all(n.startswith(f"{name}.") for n in packages[i+1:]):
# Since packages are sorted by length, this condition is able
# to find a list of all common ancestors.
# When there is divergence (e.g. multiple root packages)
# the list will be empty
break
common_ancestors.append(name)
for name in common_ancestors:
pkg_path = find_package_path(name, package_dir, root_dir)
init = os.path.join(pkg_path, "__init__.py")
if os.path.isfile(init):
return name
return None
def find_package_path(name: str, package_dir: Dict[str, str], root_dir: _Path) -> str:
"""Given a package name, return the path where it should be found on
disk, considering the ``package_dir`` option.
>>> path = find_package_path("my.pkg", {"": "root/is/nested"}, ".")
>>> path.replace(os.sep, "/")
'./root/is/nested/my/pkg'
>>> path = find_package_path("my.pkg", {"my": "root/is/nested"}, ".")
>>> path.replace(os.sep, "/")
'./root/is/nested/pkg'
>>> path = find_package_path("my.pkg", {"my.pkg": "root/is/nested"}, ".")
>>> path.replace(os.sep, "/")
'./root/is/nested'
>>> path = find_package_path("other.pkg", {"my.pkg": "root/is/nested"}, ".")
>>> path.replace(os.sep, "/")
'./other/pkg'
"""
parts = name.split(".")
for i in range(len(parts), 0, -1):
# Look backwards, the most specific package_dir first
partial_name = ".".join(parts[:i])
if partial_name in package_dir:
parent = package_dir[partial_name]
return os.path.join(root_dir, parent, *parts[i:])
parent = package_dir.get("") or ""
return os.path.join(root_dir, *parent.split("/"), *parts)
def construct_package_dir(packages: List[str], package_path: _Path) -> Dict[str, str]:
parent_pkgs = remove_nested_packages(packages)
prefix = Path(package_path).parts
return {pkg: "/".join([*prefix, *pkg.split(".")]) for pkg in parent_pkgs}
+39 -102
View File
@@ -19,7 +19,6 @@ from glob import iglob
import itertools
import textwrap
from typing import List, Optional, TYPE_CHECKING
from pathlib import Path
from collections import defaultdict
from email import message_from_file
@@ -29,10 +28,7 @@ from distutils.util import rfc822_escape
from setuptools.extern import packaging
from setuptools.extern import ordered_set
from setuptools.extern.more_itertools import unique_everseen, partition
from setuptools.extern import nspektr
from ._importlib import metadata
from setuptools.extern.more_itertools import unique_everseen
from . import SetuptoolsDeprecationWarning
@@ -40,13 +36,9 @@ import setuptools
import setuptools.command
from setuptools import windows_support
from setuptools.monkey import get_unpatched
from setuptools.config import setupcfg, pyprojecttoml
from setuptools.discovery import ConfigDiscovery
from setuptools.config import parse_configuration
import pkg_resources
from setuptools.extern.packaging import version
from . import _reqs
from . import _entry_points
if TYPE_CHECKING:
from email.message import Message
@@ -121,9 +113,13 @@ def read_pkg_file(self, file):
self.author_email = _read_field_from_msg(msg, 'author-email')
self.maintainer_email = None
self.url = _read_field_from_msg(msg, 'home-page')
self.download_url = _read_field_from_msg(msg, 'download-url')
self.license = _read_field_unescaped_from_msg(msg, 'license')
if 'download-url' in msg:
self.download_url = _read_field_from_msg(msg, 'download-url')
else:
self.download_url = None
self.long_description = _read_field_unescaped_from_msg(msg, 'description')
if (
self.long_description is None and
@@ -175,10 +171,9 @@ def write_pkg_file(self, file): # noqa: C901 # is too complex (14) # FIXME
write_field('Name', self.get_name())
write_field('Version', self.get_version())
write_field('Summary', single_line(self.get_description()))
write_field('Home-page', self.get_url())
optional_fields = (
('Home-page', 'url'),
('Download-URL', 'download_url'),
('Author', 'author'),
('Author-email', 'author_email'),
('Maintainer', 'maintainer'),
@@ -192,6 +187,8 @@ def write_pkg_file(self, file): # noqa: C901 # is too complex (14) # FIXME
license = rfc822_escape(self.get_license())
write_field('License', license)
if self.download_url:
write_field('Download-URL', self.download_url)
for project_url in self.project_urls.items():
write_field('Project-URL', '%s, %s' % project_url)
@@ -230,7 +227,7 @@ sequence = tuple, list
def check_importable(dist, attr, value):
try:
ep = metadata.EntryPoint(value=value, name=None, group=None)
ep = pkg_resources.EntryPoint.parse('x=' + value)
assert not ep.extras
except (TypeError, ValueError, AttributeError, AssertionError) as e:
raise DistutilsSetupError(
@@ -288,7 +285,7 @@ def _check_extra(extra, reqs):
name, sep, marker = extra.partition(':')
if marker and pkg_resources.invalid_marker(marker):
raise DistutilsSetupError("Invalid environment marker: " + marker)
list(_reqs.parse(reqs))
list(pkg_resources.parse_requirements(reqs))
def assert_bool(dist, attr, value):
@@ -308,7 +305,7 @@ def invalid_unless_false(dist, attr, value):
def check_requirements(dist, attr, value):
"""Verify that install_requires is a valid requirements list"""
try:
list(_reqs.parse(value))
list(pkg_resources.parse_requirements(value))
if isinstance(value, (dict, set)):
raise TypeError("Unordered types are not allowed")
except (TypeError, ValueError) as error:
@@ -333,8 +330,8 @@ def check_specifier(dist, attr, value):
def check_entry_points(dist, attr, value):
"""Verify that entry_points map is parseable"""
try:
_entry_points.load(value)
except Exception as e:
pkg_resources.EntryPoint.parse_map(value)
except ValueError as e:
raise DistutilsSetupError(e) from e
@@ -457,7 +454,7 @@ class Distribution(_Distribution):
self.patch_missing_pkg_info(attrs)
self.dependency_links = attrs.pop('dependency_links', [])
self.setup_requires = attrs.pop('setup_requires', [])
for ep in metadata.entry_points(group='distutils.setup_keywords'):
for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'):
vars(self).setdefault(ep.name, None)
_Distribution.__init__(
self,
@@ -468,13 +465,6 @@ class Distribution(_Distribution):
},
)
# Save the original dependencies before they are processed into the egg format
self._orig_extras_require = {}
self._orig_install_requires = []
self._tmp_extras_require = defaultdict(ordered_set.OrderedSet)
self.set_defaults = ConfigDiscovery(self)
self._set_metadata_defaults(attrs)
self.metadata.version = self._normalize_version(
@@ -482,19 +472,6 @@ class Distribution(_Distribution):
)
self._finalize_requires()
def _validate_metadata(self):
required = {"name"}
provided = {
key
for key in vars(self.metadata)
if getattr(self.metadata, key, None) is not None
}
missing = required - provided
if missing:
msg = f"Required package metadata is missing: {missing}"
raise DistutilsSetupError(msg)
def _set_metadata_defaults(self, attrs):
"""
Fill-in missing metadata fields not supported by distutils.
@@ -545,8 +522,6 @@ class Distribution(_Distribution):
self.metadata.python_requires = self.python_requires
if getattr(self, 'extras_require', None):
# Save original before it is messed by _convert_extras_requirements
self._orig_extras_require = self._orig_extras_require or self.extras_require
for extra in self.extras_require.keys():
# Since this gets called multiple times at points where the
# keys have become 'converted' extras, ensure that we are only
@@ -555,10 +530,6 @@ class Distribution(_Distribution):
if extra:
self.metadata.provides_extras.add(extra)
if getattr(self, 'install_requires', None) and not self._orig_install_requires:
# Save original before it is messed by _move_install_requirements_markers
self._orig_install_requires = self.install_requires
self._convert_extras_requirements()
self._move_install_requirements_markers()
@@ -569,12 +540,11 @@ class Distribution(_Distribution):
`"extra:{marker}": ["barbazquux"]`.
"""
spec_ext_reqs = getattr(self, 'extras_require', None) or {}
tmp = defaultdict(ordered_set.OrderedSet)
self._tmp_extras_require = getattr(self, '_tmp_extras_require', tmp)
self._tmp_extras_require = defaultdict(list)
for section, v in spec_ext_reqs.items():
# Do not strip empty sections.
self._tmp_extras_require[section]
for r in _reqs.parse(v):
for r in pkg_resources.parse_requirements(v):
suffix = self._suffix_for(r)
self._tmp_extras_require[section + suffix].append(r)
@@ -600,7 +570,7 @@ class Distribution(_Distribution):
return not req.marker
spec_inst_reqs = getattr(self, 'install_requires', None) or ()
inst_reqs = list(_reqs.parse(spec_inst_reqs))
inst_reqs = list(pkg_resources.parse_requirements(spec_inst_reqs))
simple_reqs = filter(is_simple_req, inst_reqs)
complex_reqs = itertools.filterfalse(is_simple_req, inst_reqs)
self.install_requires = list(map(str, simple_reqs))
@@ -608,8 +578,7 @@ class Distribution(_Distribution):
for r in complex_reqs:
self._tmp_extras_require[':' + str(r.marker)].append(r)
self.extras_require = dict(
# list(dict.fromkeys(...)) ensures a list of unique strings
(k, list(dict.fromkeys(str(r) for r in map(self._clean_req, v))))
(k, [str(r) for r in map(self._clean_req, v)])
for k, v in self._tmp_extras_require.items()
)
@@ -742,10 +711,7 @@ class Distribution(_Distribution):
return opt
underscore_opt = opt.replace('-', '_')
commands = list(itertools.chain(
distutils.command.__all__,
self._setuptools_commands(),
))
commands = distutils.command.__all__ + self._setuptools_commands()
if (
not section.startswith('options')
and section != 'metadata'
@@ -763,8 +729,9 @@ class Distribution(_Distribution):
def _setuptools_commands(self):
try:
return metadata.distribution('setuptools').entry_points.names
except metadata.PackageNotFoundError:
dist = pkg_resources.get_distribution('setuptools')
return list(dist.get_entry_map('distutils.commands'))
except pkg_resources.DistributionNotFound:
# during bootstrapping, distribution doesn't exist
return []
@@ -827,39 +794,23 @@ class Distribution(_Distribution):
except ValueError as e:
raise DistutilsOptionError(e) from e
def _get_project_config_files(self, filenames):
"""Add default file and split between INI and TOML"""
tomlfiles = []
standard_project_metadata = Path(self.src_root or os.curdir, "pyproject.toml")
if filenames is not None:
parts = partition(lambda f: Path(f).suffix == ".toml", filenames)
filenames = list(parts[0]) # 1st element => predicate is False
tomlfiles = list(parts[1]) # 2nd element => predicate is True
elif standard_project_metadata.exists():
tomlfiles = [standard_project_metadata]
return filenames, tomlfiles
def parse_config_files(self, filenames=None, ignore_option_errors=False):
"""Parses configuration files from various levels
and loads configuration.
"""
inifiles, tomlfiles = self._get_project_config_files(filenames)
self._parse_config_files(filenames=filenames)
self._parse_config_files(filenames=inifiles)
setupcfg.parse_configuration(
parse_configuration(
self, self.command_options, ignore_option_errors=ignore_option_errors
)
for filename in tomlfiles:
pyprojecttoml.apply_configuration(self, filename, ignore_option_errors)
self._finalize_requires()
self._finalize_license_files()
def fetch_build_eggs(self, requires):
"""Resolve pre-setup requirements"""
resolved_dists = pkg_resources.working_set.resolve(
_reqs.parse(requires),
pkg_resources.parse_requirements(requires),
installer=self.fetch_build_egg,
replace_conflicting=True,
)
@@ -879,7 +830,7 @@ class Distribution(_Distribution):
def by_order(hook):
return getattr(hook, 'order', 0)
defined = metadata.entry_points(group=group)
defined = pkg_resources.iter_entry_points(group)
filtered = itertools.filterfalse(self._removed, defined)
loaded = map(lambda e: e.load(), filtered)
for ep in sorted(loaded, key=by_order):
@@ -900,21 +851,12 @@ class Distribution(_Distribution):
return ep.name in removed
def _finalize_setup_keywords(self):
for ep in metadata.entry_points(group='distutils.setup_keywords'):
for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'):
value = getattr(self, ep.name, None)
if value is not None:
self._install_dependencies(ep)
ep.require(installer=self.fetch_build_egg)
ep.load()(self, ep.name, value)
def _install_dependencies(self, ep):
"""
Given an entry point, ensure that any declared extras for
its distribution are installed.
"""
for req in nspektr.missing(ep):
# fetch_build_egg expects pkg_resources.Requirement
self.fetch_build_egg(pkg_resources.Requirement(str(req)))
def get_egg_cache_dir(self):
egg_cache_dir = os.path.join(os.curdir, '.eggs')
if not os.path.exists(egg_cache_dir):
@@ -945,25 +887,27 @@ class Distribution(_Distribution):
if command in self.cmdclass:
return self.cmdclass[command]
eps = metadata.entry_points(group='distutils.commands', name=command)
eps = pkg_resources.iter_entry_points('distutils.commands', command)
for ep in eps:
self._install_dependencies(ep)
ep.require(installer=self.fetch_build_egg)
self.cmdclass[command] = cmdclass = ep.load()
return cmdclass
else:
return _Distribution.get_command_class(self, command)
def print_commands(self):
for ep in metadata.entry_points(group='distutils.commands'):
for ep in pkg_resources.iter_entry_points('distutils.commands'):
if ep.name not in self.cmdclass:
cmdclass = ep.load()
# don't require extras as the commands won't be invoked
cmdclass = ep.resolve()
self.cmdclass[ep.name] = cmdclass
return _Distribution.print_commands(self)
def get_command_list(self):
for ep in metadata.entry_points(group='distutils.commands'):
for ep in pkg_resources.iter_entry_points('distutils.commands'):
if ep.name not in self.cmdclass:
cmdclass = ep.load()
# don't require extras as the commands won't be invoked
cmdclass = ep.resolve()
self.cmdclass[ep.name] = cmdclass
return _Distribution.get_command_list(self)
@@ -1206,13 +1150,6 @@ class Distribution(_Distribution):
sys.stdout.detach(), encoding, errors, newline, line_buffering
)
def run_command(self, command):
self.set_defaults()
# Postpone defaults until all explicit configuration is considered
# (setup() args, config files, command line and plugins)
super().run_command(command)
class DistDeprecationWarning(SetuptoolsDeprecationWarning):
"""Class for warning about deprecations in dist in
+11 -29
View File
@@ -4,6 +4,17 @@ Provides exceptions used by setuptools modules.
"""
from distutils import errors as _distutils_errors
from distutils.errors import DistutilsError
class RemovedCommandError(DistutilsError, RuntimeError):
"""Error used for commands that have been removed in setuptools.
Since ``setuptools`` is built on ``distutils``, simply removing a command
from ``setuptools`` will make the behavior fall back to ``distutils``; this
error is raised if a command exists in ``distutils`` but has been actively
removed in ``setuptools``.
"""
# Re-export errors from distutils to facilitate the migration to PEP632
@@ -27,32 +38,3 @@ UnknownFileError = _distutils_errors.UnknownFileError
# The root error class in the hierarchy
BaseError = _distutils_errors.DistutilsError
class RemovedCommandError(BaseError, RuntimeError):
"""Error used for commands that have been removed in setuptools.
Since ``setuptools`` is built on ``distutils``, simply removing a command
from ``setuptools`` will make the behavior fall back to ``distutils``; this
error is raised if a command exists in ``distutils`` but has been actively
removed in ``setuptools``.
"""
class PackageDiscoveryError(BaseError, RuntimeError):
"""Impossible to perform automatic discovery of packages and/or modules.
The current project layout or given discovery options can lead to problems when
scanning the project directory.
Setuptools might also refuse to complete auto-discovery if an error prone condition
is detected (e.g. when a project is organised as a flat-layout but contains
multiple directories that can be taken as top-level packages inside a single
distribution [*]_). In these situations the users are encouraged to be explicit
about which packages to include or to make the discovery parameters more specific.
.. [*] Since multi-package distributions are uncommon it is very likely that the
developers did not intend for all the directories to be packaged, and are just
leaving auxiliary code in the repository top-level, such as maintenance-related
scripts.
"""
@@ -34,7 +34,7 @@ class Extension(_Extension):
# The *args is needed for compatibility as calls may use positional
# arguments. py_limited_api may be set only via keyword.
self.py_limited_api = kw.pop("py_limited_api", False)
super().__init__(name, sources, *args, **kw)
_Extension.__init__(self, name, sources, *args, **kw)
def _convert_pyx_sources_to_lang(self):
"""
+1 -4
View File
@@ -69,8 +69,5 @@ class VendorImporter:
sys.meta_path.append(self)
names = (
'packaging', 'pyparsing', 'ordered_set', 'more_itertools', 'importlib_metadata',
'zipp', 'importlib_resources', 'jaraco', 'typing_extensions', 'nspektr', 'tomli',
)
names = 'packaging', 'pyparsing', 'ordered_set', 'more_itertools',
VendorImporter(__name__, names, 'setuptools._vendor').install()
@@ -24,12 +24,6 @@ def configure():
format="{message}", style='{', handlers=handlers, level=logging.DEBUG)
monkey.patch_func(set_threshold, distutils.log, 'set_threshold')
# For some reason `distutils.log` module is getting cached in `distutils.dist`
# and then loaded again when patched,
# implying: id(distutils.log) != id(distutils.dist.log).
# Make sure the same module object is used everywhere:
distutils.dist.log = distutils.log
def set_threshold(level):
logging.root.setLevel(level*10)
@@ -285,7 +285,7 @@ class PackageIndex(Environment):
self, index_url="https://pypi.org/simple/", hosts=('*',),
ca_bundle=None, verify_ssl=True, *args, **kw
):
super().__init__(*args, **kw)
Environment.__init__(self, *args, **kw)
self.index_url = index_url + "/" [:not index_url.endswith('/')]
self.scanned_urls = {}
self.fetched_urls = {}
@@ -680,7 +680,8 @@ class PackageIndex(Environment):
# Make sure the file has been downloaded to the temp dir.
if os.path.dirname(filename) != tmpdir:
dst = os.path.join(tmpdir, basename)
if not (os.path.exists(dst) and os.path.samefile(filename, dst)):
from setuptools.command.easy_install import samefile
if not samefile(filename, dst):
shutil.copy2(filename, dst)
filename = dst
@@ -1001,7 +1002,7 @@ class PyPIConfig(configparser.RawConfigParser):
Load from ~/.pypirc
"""
defaults = dict.fromkeys(['username', 'password', 'repository'], '')
super().__init__(defaults)
configparser.RawConfigParser.__init__(self, defaults)
rc = os.path.join(os.path.expanduser('~'), '.pypirc')
if os.path.exists(rc):
+5 -5
View File
@@ -15,7 +15,6 @@ from pkg_resources import parse_version
from setuptools.extern.packaging.tags import sys_tags
from setuptools.extern.packaging.utils import canonicalize_name
from setuptools.command.egg_info import write_requirements
from setuptools.archive_util import _unpack_zipfile_obj
WHEEL_NAME = re.compile(
@@ -122,7 +121,8 @@ class Wheel:
raise ValueError(
'unsupported wheel format version: %s' % wheel_version)
# Extract to target directory.
_unpack_zipfile_obj(zf, destination_eggdir)
os.mkdir(destination_eggdir)
zf.extractall(destination_eggdir)
# Convert metadata.
dist_info = os.path.join(destination_eggdir, dist_info)
dist = pkg_resources.Distribution.from_location(
@@ -136,13 +136,13 @@ class Wheel:
def raw_req(req):
req.marker = None
return str(req)
install_requires = list(map(raw_req, dist.requires()))
install_requires = list(sorted(map(raw_req, dist.requires())))
extras_require = {
extra: [
extra: sorted(
req
for req in map(raw_req, dist.requires((extra,)))
if req not in install_requires
]
)
for extra in dist.extras
}
os.rename(dist_info, egg_info)
@@ -1,4 +1,5 @@
import platform
import ctypes
def windows_only(func):
@@ -16,7 +17,6 @@ def hide_file(path):
`path` must be text.
"""
import ctypes
__import__('ctypes.wintypes')
SetFileAttributes = ctypes.windll.kernel32.SetFileAttributesW
SetFileAttributes.argtypes = ctypes.wintypes.LPWSTR, ctypes.wintypes.DWORD