[WIP] Progressive work towards working pip again (#8)

* Minimal hack to make install work again

* Add NotImplementedError to branches not handling variants right now

* Disentangle variants from tags

Remove the code responsible for handling variants via injecting them
into tags.  This is in preparation for a new logic that is going
to handle variants more explicitly.

* Remove unused `read_provider_priority_from_pip_config()`

* Reimplement variant filtering and sorting directly

* Fetch variants.json lazily

* Support multiple variants.json files

* Fix variant priorities

* Add missing sys import

* Let variantlib take care of plugin loading

* Fix installing from local files

* Remove stale PluginLoader import

Author: mgorny

src/pip/_internal/cache.py

@@ -150,6 +150,7 @@ def get(
                     package_name,
                 )
                 continue
+            raise NotImplementedError
             if not wheel.supported(supported_tags):
                 # Built for a different python/arch/etc
                 continue

src/pip/_internal/index/package_finder.py

@@ -5,6 +5,7 @@
 import itertools
 import logging
 import re
+import sys
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, FrozenSet, Iterable, List, Optional, Set, Tuple, Union
 
@@ -36,7 +37,11 @@
 from pip._internal.utils.misc import build_netloc
 from pip._internal.utils.packaging import check_requires_python
 from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS
-from pip._internal.utils.variant import VariantJson
+from pip._internal.utils.variant import (
+    VariantJson,
+    get_cached_variant_hashes_by_priority,
+    get_variants_json_filename,
+)
 
 if TYPE_CHECKING:
     from pip._vendor.typing_extensions import TypeGuard
@@ -105,6 +110,7 @@ class LinkType(enum.Enum):
     format_invalid = enum.auto()
     platform_mismatch = enum.auto()
     requires_python_mismatch = enum.auto()
+    variant_unsupported = enum.auto()
 
 
 class LinkEvaluator:
@@ -154,7 +160,7 @@ def __init__(
         self._target_python = target_python
 
         self.project_name = project_name
-        self.variants_json = None
+        self.variants_json = {}
 
     def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]:
         """
@@ -204,9 +210,7 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]:
                     return (LinkType.different_project, reason, None)
 
                 variant_hash = wheel.variant_hash
-                supported_tags = self._target_python.get_unsorted_tags(
-                    variants_json=self.variants_json
-                )
+                supported_tags = self._target_python.get_unsorted_tags()
                 if not wheel.supported(supported_tags):
                     # Include the wheel's tags in the reason string to
                     # simplify troubleshooting compatibility issues.
@@ -217,6 +221,20 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]:
                     )
                     return (LinkType.platform_mismatch, reason, None)
 
+                supported_variants = set(
+                    get_cached_variant_hashes_by_priority(
+                        self.variants_json.get(
+                            get_variants_json_filename(wheel)
+                        )
+                    )
+                )
+                if wheel.variant_hash not in supported_variants:
+                    reason = (
+                        f"variant {wheel.variant_hash} is not compatible with "
+                        f"the system"
+                    )
+                    return (LinkType.variant_unsupported, reason, None)
+
                 version = wheel.version
 
         # This should be up by the self.ok_binary check, but see issue 2700.
@@ -384,7 +402,7 @@ def create(
         allow_all_prereleases: bool = False,
         specifier: Optional[specifiers.BaseSpecifier] = None,
         hashes: Optional[Hashes] = None,
-        variants_json: Optional[VariantJson] = None,
+        variants_json: dict[VariantJson] = {},
     ) -> "CandidateEvaluator":
         """Create a CandidateEvaluator object.
 
@@ -401,9 +419,7 @@ def create(
         if specifier is None:
             specifier = specifiers.SpecifierSet()
 
-        supported_tags = target_python.get_sorted_tags(
-            variants_json=variants_json,
-        )
+        supported_tags = target_python.get_sorted_tags()
 
         return cls(
             project_name=project_name,
@@ -412,6 +428,7 @@ def create(
             prefer_binary=prefer_binary,
             allow_all_prereleases=allow_all_prereleases,
             hashes=hashes,
+            variants_json=variants_json,
         )
 
     def __init__(
@@ -422,6 +439,7 @@ def __init__(
         prefer_binary: bool = False,
         allow_all_prereleases: bool = False,
         hashes: Optional[Hashes] = None,
+        variants_json: dict[VariantJson] = [],
     ) -> None:
         """
         :param supported_tags: The PEP 425 tags supported by the target
@@ -433,6 +451,7 @@ def __init__(
         self._project_name = project_name
         self._specifier = specifier
         self._supported_tags = supported_tags
+        self._variants_json = variants_json
         # Since the index of the tag in the _supported_tags list is used
         # as a priority, precompute a map from tag to index/priority to be
         # used in wheel.find_most_preferred_tag.
@@ -513,12 +532,20 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey:
         if link.is_wheel:
             # can raise InvalidWheelFilename
             wheel = Wheel(link.filename)
+
+            supported_variants = get_cached_variant_hashes_by_priority(
+                self._variants_json.get(
+                    get_variants_json_filename(wheel)
+                )
+            )
+
             try:
                 pri = -(
                     wheel.find_most_preferred_tag(
                         valid_tags, self._wheel_tag_preferences
                     )
                 )
+                variant_pri = -supported_variants.index(wheel.variant_hash)
             except ValueError:
                 raise UnsupportedWheel(
                     f"{wheel.filename} is not a supported wheel for this platform. It "
@@ -533,13 +560,15 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey:
                 build_tag = (int(build_tag_groups[0]), build_tag_groups[1])
         else:  # sdist
             pri = -(support_num)
+            variant_pri = -sys.maxsize
         has_allowed_hash = int(link.is_hash_allowed(self._hashes))
         yank_value = -1 * int(link.is_yanked)  # -1 for yanked.
         return (
             has_allowed_hash,
             yank_value,
             binary_preference,
             candidate.version,
+            variant_pri,
             pri,
             build_tag,
         )
@@ -741,7 +770,7 @@ def _sort_links(self, links: Iterable[Link]) -> List[Link]:
         for link in links:
             if link not in seen:
                 seen.add(link)
-                if link.filename == "variants.json":
+                if link.filename.endswith("-variants.json"):
                     variants_json.append(link)
                 elif link.egg_fragment:
                     eggs.append(link)
@@ -784,10 +813,6 @@ def get_install_candidate(
         except InvalidVersion:
             return None
 
-    @functools.cache
-    def get_variants_json(self, link: Link) -> dict:
-        return VariantJson(self._link_collector.session.request("GET", link.url).json())
-
     def evaluate_links(
         self, link_evaluator: LinkEvaluator, links: Iterable[Link]
     ) -> List[InstallationCandidate]:
@@ -796,8 +821,11 @@ def evaluate_links(
         """
         candidates = []
         for link in self._sort_links(links):
-            if link.filename == "variants.json":
-                link_evaluator.variants_json = self.get_variants_json(link)
+            if link.filename.endswith("-variants.json"):
+                link_evaluator.variants_json[link.filename] = VariantJson(
+                    link.url,
+                    lambda url: self._link_collector.session.request("GET", url).json(),
+                )
                 continue
 
             candidate = self.get_install_candidate(link_evaluator, link)

src/pip/_internal/models/target_python.py

@@ -81,7 +81,6 @@ def format_given(self) -> str:
         )
 
     def get_sorted_tags(self,
-                        variants_json: Optional[VariantJson] = None
                         ) -> List[Tag]:
         """
         Return the supported PEP 425 tags to check wheel candidates against.
@@ -101,16 +100,12 @@ def get_sorted_tags(self,
             platforms=self.platforms,
             abis=self.abis,
             impl=self.implementation,
-            variants_json=variants_json,
         )
 
     def get_unsorted_tags(self,
-                          variants_json: Optional[VariantJson] = None
                           ) -> Set[Tag]:
         """Exactly the same as get_sorted_tags, but returns a set.
 
         This is important for performance.
         """
-        return set(self.get_sorted_tags(
-            variants_json=variants_json,
-        ))
+        return set(self.get_sorted_tags())

src/pip/_internal/models/wheel.py

@@ -66,7 +66,7 @@ def __init__(self, filename: str) -> None:
 
         # All the tag combinations from this file
         self.file_tags = {
-            Tag(x, y, z, self.variant_hash)
+            Tag(x, y, z)
             for x in self.pyversions
             for y in self.abis
             for z in self.plats

src/pip/_internal/resolution/legacy/resolver.py

@@ -225,6 +225,7 @@ def _add_requirement_to_set(
         if install_req.link and install_req.link.is_wheel:
             wheel = Wheel(install_req.link.filename)
             tags = compatibility_tags.get_supported()
+            raise NotImplementedError
             if requirement_set.check_supported_wheels and not wheel.supported(tags):
                 raise InstallationError(
                     f"{wheel.filename} is not a supported wheel on this platform."

src/pip/_internal/resolution/resolvelib/factory.py

@@ -53,6 +53,7 @@
 from pip._internal.utils.compatibility_tags import get_supported
 from pip._internal.utils.hashes import Hashes
 from pip._internal.utils.packaging import get_requirement
+from pip._internal.utils.variant import variant_wheel_supported
 from pip._internal.utils.virtualenv import running_under_virtualenv
 
 from .base import Candidate, Constraint, Requirement
@@ -141,7 +142,10 @@ def _fail_if_link_is_unsupported_wheel(self, link: Link) -> None:
         if not link.is_wheel:
             return
         wheel = Wheel(link.filename)
-        if wheel.supported(self._finder.target_python.get_unsorted_tags()):
+        if (
+            wheel.supported(self._finder.target_python.get_unsorted_tags())
+            and variant_wheel_supported(wheel, link)
+        ):
             return
         msg = f"{link.filename} is not a supported wheel on this platform."
         raise UnsupportedWheel(msg)

src/pip/_internal/utils/compatibility_tags.py

@@ -17,11 +17,6 @@
     mac_platforms,
 )
 
-from pip._internal.utils.variant import (
-    VariantJson,
-    get_cached_variant_hashes_by_priority,
-)
-
 _apple_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)")
 
 
@@ -147,7 +142,6 @@ def get_supported(
     platforms: Optional[List[str]] = None,
     impl: Optional[str] = None,
     abis: Optional[List[str]] = None,
-    variants_json: Optional[VariantJson] = None,
 ) -> List[Tag]:
     """Return a list of supported tags for each version specified in
     `versions`.
@@ -196,41 +190,4 @@ def get_supported(
         )
     )
 
-    if variants_json is not None:
-        variants_by_priority = get_cached_variant_hashes_by_priority(
-            variants_json=variants_json
-        )
-
-        # NOTE: There is two choices implementation wise
-        # QUESTION: Which one should be the outer loop ?
-        #
-        # 1. Shall it iterate over all variants per `Tag`:
-        #   - cp313-cp313-manylinux_2_36_aarch64-054fcdf8
-        #   - cp313-cp313-manylinux_2_35_aarch64-054fcdf8
-        #   - cp313-cp313-manylinux_2_34_aarch64-054fcdf8
-        #   - ...
-        #   - cp313-cp313-manylinux_2_36_aarch64-39614353
-        #   - cp313-cp313-manylinux_2_35_aarch64-39614353
-        #   - cp313-cp313-manylinux_2_34_aarch64-39614353
-        #   - ...
-        #
-        # 2. Shall it iterate over all tags per `Variants`:
-        #   - cp313-cp313-manylinux_2_36_aarch64-054fcdf8
-        #   - cp313-cp313-manylinux_2_36_aarch64-39614353
-        #   - ...
-        #   - cp313-cp313-manylinux_2_35_aarch64-054fcdf8
-        #   - cp313-cp313-manylinux_2_35_aarch64-39614353
-        #   - ...
-        #   - cp313-cp313-manylinux_2_34_aarch64-054fcdf8
-        #   - cp313-cp313-manylinux_2_34_aarch64-39614353
-        #   - ...
-
-        # Current implementation is choice 1)
-        # Flip the order of `for loops` to switch to choice 2)
-        supported = [
-            Tag.create_varianttag_from_tag(tag, variant_hash=variant_hash)
-            for variant_hash in variants_by_priority
-            for tag in supported
-        ] + supported
-
     return supported