Skip to content

Commit 9ba2bb9

Browse files
committed
Straighten up extra comps across metadata backends
The importlib.metadata and pkg_resources backends unfortunately normalize extras differently, and we don't really want to continue using the latter's logic (being partially lossy while still not compliant to standards), so we add a new abstraction for the purpose.
1 parent 7127fc9 commit 9ba2bb9

File tree

7 files changed

+64
-22
lines changed

7 files changed

+64
-22
lines changed

src/pip/_internal/metadata/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from .base import BaseDistribution, BaseEnvironment, FilesystemWheel, MemoryWheel, Wheel
1010

1111
if TYPE_CHECKING:
12-
from typing import Protocol
12+
from typing import Literal, Protocol
1313
else:
1414
Protocol = object
1515

@@ -50,6 +50,7 @@ def _should_use_importlib_metadata() -> bool:
5050

5151

5252
class Backend(Protocol):
53+
NAME: 'Literal["importlib", "pkg_resources"]'
5354
Distribution: Type[BaseDistribution]
5455
Environment: Type[BaseEnvironment]
5556

src/pip/_internal/metadata/base.py

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525
from pip._vendor.packaging.requirements import Requirement
2626
from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
27-
from pip._vendor.packaging.utils import NormalizedName
27+
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
2828
from pip._vendor.packaging.version import LegacyVersion, Version
2929

3030
from pip._internal.exceptions import NoneMetadataError
@@ -37,7 +37,6 @@
3737
from pip._internal.utils.compat import stdlib_pkgs # TODO: Move definition here.
3838
from pip._internal.utils.egg_link import egg_link_path_from_sys_path
3939
from pip._internal.utils.misc import is_local, normalize_path
40-
from pip._internal.utils.packaging import safe_extra
4140
from pip._internal.utils.urls import url_to_path
4241

4342
from ._json import msg_to_json
@@ -460,6 +459,19 @@ def iter_provided_extras(self) -> Iterable[str]:
460459
461460
For modern .dist-info distributions, this is the collection of
462461
"Provides-Extra:" entries in distribution metadata.
462+
463+
The return value of this function is not particularly useful other than
464+
display purposes due to backward compatibility issues and the extra
465+
names being poorly normalized prior to PEP 685. If you want to perform
466+
logic operations on extras, use :func:`is_extra_provided` instead.
467+
"""
468+
raise NotImplementedError()
469+
470+
def is_extra_provided(self, extra: str) -> bool:
471+
"""Check whether an extra is provided by this distribution.
472+
473+
This is needed mostly for compatibility issues with pkg_resources not
474+
following the extra normalization rules defined in PEP 685.
463475
"""
464476
raise NotImplementedError()
465477

@@ -537,10 +549,11 @@ def _iter_egg_info_extras(self) -> Iterable[str]:
537549
"""Get extras from the egg-info directory."""
538550
known_extras = {""}
539551
for entry in self._iter_requires_txt_entries():
540-
if entry.extra in known_extras:
552+
extra = canonicalize_name(entry.extra)
553+
if extra in known_extras:
541554
continue
542-
known_extras.add(entry.extra)
543-
yield entry.extra
555+
known_extras.add(extra)
556+
yield extra
544557

545558
def _iter_egg_info_dependencies(self) -> Iterable[str]:
546559
"""Get distribution dependencies from the egg-info directory.
@@ -556,10 +569,11 @@ def _iter_egg_info_dependencies(self) -> Iterable[str]:
556569
all currently available PEP 517 backends, although not standardized.
557570
"""
558571
for entry in self._iter_requires_txt_entries():
559-
if entry.extra and entry.marker:
560-
marker = f'({entry.marker}) and extra == "{safe_extra(entry.extra)}"'
561-
elif entry.extra:
562-
marker = f'extra == "{safe_extra(entry.extra)}"'
572+
extra = canonicalize_name(entry.extra)
573+
if extra and entry.marker:
574+
marker = f'({entry.marker}) and extra == "{extra}"'
575+
elif extra:
576+
marker = f'extra == "{extra}"'
563577
elif entry.marker:
564578
marker = entry.marker
565579
else:
Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
from ._dists import Distribution
22
from ._envs import Environment
33

4-
__all__ = ["Distribution", "Environment"]
4+
__all__ = ["NAME", "Distribution", "Environment"]
5+
6+
NAME = "importlib"

src/pip/_internal/metadata/importlib/_dists.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,13 @@ def _metadata_impl(self) -> email.message.Message:
207207
return cast(email.message.Message, self._dist.metadata)
208208

209209
def iter_provided_extras(self) -> Iterable[str]:
210-
return (extra for extra in self.metadata.get_all("Provides-Extra", []))
210+
return self.metadata.get_all("Provides-Extra", [])
211+
212+
def is_extra_provided(self, extra: str) -> bool:
213+
return any(
214+
canonicalize_name(provided_extra) == canonicalize_name(extra)
215+
for provided_extra in self.metadata.get_all("Provides-Extra", [])
216+
)
211217

212218
def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
213219
contexts: Sequence[Dict[str, str]] = [{"extra": e} for e in extras]

src/pip/_internal/metadata/pkg_resources.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,12 @@
2424
Wheel,
2525
)
2626

27+
__all__ = ["NAME", "Distribution", "Environment"]
28+
2729
logger = logging.getLogger(__name__)
2830

31+
NAME = "pkg_resources"
32+
2933

3034
class EntryPoint(NamedTuple):
3135
name: str
@@ -212,12 +216,16 @@ def _metadata_impl(self) -> email.message.Message:
212216

213217
def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
214218
if extras: # pkg_resources raises on invalid extras, so we sanitize.
215-
extras = frozenset(extras).intersection(self._dist.extras)
219+
extras = frozenset(pkg_resources.safe_extra(e) for e in extras)
220+
extras = extras.intersection(self._dist.extras)
216221
return self._dist.requires(extras)
217222

218223
def iter_provided_extras(self) -> Iterable[str]:
219224
return self._dist.extras
220225

226+
def is_extra_provided(self, extra: str) -> bool:
227+
return pkg_resources.safe_extra(extra) in self._dist.extras
228+
221229

222230
class Environment(BaseEnvironment):
223231
def __init__(self, ws: pkg_resources.WorkingSet) -> None:

src/pip/_internal/req/req_install.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,11 @@ def match_markers(self, extras_requested: Optional[Iterable[str]] = None) -> boo
272272
extras_requested = ("",)
273273
if self.markers is not None:
274274
return any(
275-
self.markers.evaluate({"extra": safe_extra(extra)})
275+
self.markers.evaluate({"extra": extra})
276+
# TODO: Remove these two variants when packaging is upgraded to
277+
# support the marker comparison logic specified in PEP 685.
278+
or self.markers.evaluate({"extra": safe_extra(extra)})
279+
or self.markers.evaluate({"extra": canonicalize_name(extra)})
276280
for extra in extras_requested
277281
)
278282
else:

src/pip/_internal/resolution/resolvelib/candidates.py

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -435,7 +435,8 @@ def __init__(
435435
# since PEP 685 has not been implemented for marker-matching, and using
436436
# the non-normalized extra for lookup ensures the user can select a
437437
# non-normalized extra in a package with its non-normalized form.
438-
# TODO: Remove this when packaging is upgraded to support PEP 685.
438+
# TODO: Remove this attribute when packaging is upgraded to support the
439+
# marker comparison logic specified in PEP 685.
439440
self._unnormalized_extras = extras.difference(self.extras)
440441

441442
def __str__(self) -> str:
@@ -490,18 +491,20 @@ def source_link(self) -> Optional[Link]:
490491
def _warn_invalid_extras(
491492
self,
492493
requested: FrozenSet[str],
493-
provided: FrozenSet[str],
494+
valid: FrozenSet[str],
494495
) -> None:
495496
"""Emit warnings for invalid extras being requested.
496497
497498
This emits a warning for each requested extra that is not in the
498499
candidate's ``Provides-Extra`` list.
499500
"""
500-
invalid_extras_to_warn = requested.difference(
501-
provided,
501+
invalid_extras_to_warn = frozenset(
502+
extra
503+
for extra in requested
504+
if extra not in valid
502505
# If an extra is requested in an unnormalized form, skip warning
503506
# about the normalized form being missing.
504-
(canonicalize_name(e) for e in self._unnormalized_extras),
507+
and extra in self.extras
505508
)
506509
if not invalid_extras_to_warn:
507510
return
@@ -521,9 +524,13 @@ def _calculate_valid_requested_extras(self) -> FrozenSet[str]:
521524
cause a warning to be logged here.
522525
"""
523526
requested_extras = self.extras.union(self._unnormalized_extras)
524-
provided_extras = frozenset(self.base.dist.iter_provided_extras())
525-
self._warn_invalid_extras(requested_extras, provided_extras)
526-
return requested_extras.intersection(provided_extras)
527+
valid_extras = frozenset(
528+
extra
529+
for extra in requested_extras
530+
if self.base.dist.is_extra_provided(extra)
531+
)
532+
self._warn_invalid_extras(requested_extras, valid_extras)
533+
return valid_extras
527534

528535
def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]:
529536
factory = self.base._factory

0 commit comments

Comments
 (0)