Merge pull request #2013 from AndydeCleyre/bugfix/2004

Ensure consistent extras formatting in output

Author: chrysle

piptools/_compat/__init__.py

@@ -1,15 +1,13 @@
 from __future__ import annotations
 
 from .pip_compat import (
-    PIP_VERSION,
     Distribution,
     create_wheel_cache,
     get_dev_pkgs,
     parse_requirements,
 )
 
 __all__ = [
-    "PIP_VERSION",
     "Distribution",
     "parse_requirements",
     "create_wheel_cache",

piptools/_compat/pip_compat.py

@@ -4,7 +4,6 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Iterable, Iterator, Set, cast
 
-import pip
 from pip._internal.cache import WheelCache
 from pip._internal.index.package_finder import PackageFinder
 from pip._internal.metadata import BaseDistribution
@@ -15,18 +14,17 @@
 from pip._internal.req import InstallRequirement
 from pip._internal.req import parse_requirements as _parse_requirements
 from pip._internal.req.constructors import install_req_from_parsed_requirement
-from pip._vendor.packaging.version import parse as parse_version
 from pip._vendor.pkg_resources import Requirement
 
-PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split(".")))
-
 # The Distribution interface has changed between pkg_resources and
 # importlib.metadata, so this compat layer allows for a consistent access
 # pattern. In pip 22.1, importlib.metadata became the default on Python 3.11
 # (and later), but is overridable. `select_backend` returns what's being used.
 if TYPE_CHECKING:
     from pip._internal.metadata.importlib import Distribution as _ImportLibDist
 
+from ..utils import PIP_VERSION, copy_install_requirement
+
 
 @dataclass(frozen=True)
 class Distribution:
@@ -91,7 +89,7 @@ def parse_requirements(
             file_link = FileLink(install_req.link.url)
             file_link._url = parsed_req.requirement
             install_req.link = file_link
-        yield install_req
+        yield copy_install_requirement(install_req)
 
 
 def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache:

piptools/build.py

@@ -13,10 +13,12 @@
 import build.env
 import pyproject_hooks
 from pip._internal.req import InstallRequirement
-from pip._internal.req.constructors import install_req_from_line, parse_req_from_line
+from pip._internal.req.constructors import parse_req_from_line
 from pip._vendor.packaging.markers import Marker
 from pip._vendor.packaging.requirements import Requirement
 
+from .utils import copy_install_requirement, install_req_from_line
+
 if sys.version_info >= (3, 11):
     import tomllib
 else:
@@ -229,12 +231,14 @@ def _prepare_requirements(
             replaced_package_name = req.replace(package_name, str(package_dir), 1)
             parts = parse_req_from_line(replaced_package_name, comes_from)
 
-        yield InstallRequirement(
-            parts.requirement,
-            comes_from,
-            link=parts.link,
-            markers=parts.markers,
-            extras=parts.extras,
+        yield copy_install_requirement(
+            InstallRequirement(
+                parts.requirement,
+                comes_from,
+                link=parts.link,
+                markers=parts.markers,
+                extras=parts.extras,
+            )
         )
 
 

piptools/resolver.py

@@ -14,7 +14,6 @@
     update_env_context_manager,
 )
 from pip._internal.req import InstallRequirement
-from pip._internal.req.constructors import install_req_from_line
 from pip._internal.resolution.resolvelib.base import Candidate
 from pip._internal.resolution.resolvelib.candidates import ExtrasCandidate
 from pip._internal.resolution.resolvelib.resolver import Resolver
@@ -36,6 +35,7 @@
     copy_install_requirement,
     format_requirement,
     format_specifier,
+    install_req_from_line,
     is_pinned_requirement,
     is_url_requirement,
     key_from_ireq,

piptools/scripts/compile.py

@@ -12,7 +12,6 @@
 from build import BuildBackendException
 from click.utils import LazyFile, safecall
 from pip._internal.req import InstallRequirement
-from pip._internal.req.constructors import install_req_from_line
 from pip._internal.utils.misc import redact_auth_from_url
 
 from .._compat import parse_requirements
@@ -23,7 +22,13 @@
 from ..repositories import LocalRequirementsRepository, PyPIRepository
 from ..repositories.base import BaseRepository
 from ..resolver import BacktrackingResolver, LegacyResolver
-from ..utils import dedup, drop_extras, is_pinned_requirement, key_from_ireq
+from ..utils import (
+    dedup,
+    drop_extras,
+    install_req_from_line,
+    is_pinned_requirement,
+    key_from_ireq,
+)
 from ..writer import OutputWriter
 from . import options
 from .options import BuildTargetT

piptools/utils.py

@@ -20,9 +20,12 @@
     import tomli as tomllib
 
 import click
+import pip
 from click.utils import LazyFile
 from pip._internal.req import InstallRequirement
-from pip._internal.req.constructors import install_req_from_line
+from pip._internal.req.constructors import (
+    install_req_from_line as _install_req_from_line,
+)
 from pip._internal.resolution.resolvelib.base import Requirement as PipRequirement
 from pip._internal.utils.misc import redact_auth_from_url
 from pip._internal.vcs import is_url
@@ -31,9 +34,9 @@
 from pip._vendor.packaging.specifiers import SpecifierSet
 from pip._vendor.packaging.utils import canonicalize_name
 from pip._vendor.packaging.version import Version
+from pip._vendor.packaging.version import parse as parse_version
 from pip._vendor.pkg_resources import get_distribution
 
-from piptools._compat import PIP_VERSION
 from piptools.locations import DEFAULT_CONFIG_FILE_NAMES
 from piptools.subprocess_utils import run_python_snippet
 
@@ -42,6 +45,8 @@
 _T = TypeVar("_T")
 _S = TypeVar("_S")
 
+PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split(".")))
+
 UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"}
 COMPILE_EXCLUDE_OPTIONS = {
     "--dry-run",
@@ -88,6 +93,10 @@ def comment(text: str) -> str:
     return click.style(text, fg="green")
 
 
+def install_req_from_line(*args: Any, **kwargs: Any) -> InstallRequirement:
+    return copy_install_requirement(_install_req_from_line(*args, **kwargs))
+
+
 def make_install_requirement(
     name: str, version: str | Version, ireq: InstallRequirement
 ) -> InstallRequirement:
@@ -515,6 +524,10 @@ def copy_install_requirement(
     if "req" not in kwargs:
         kwargs["req"] = copy.deepcopy(template.req)
 
+    kwargs["extras"] = set(map(canonicalize_name, kwargs["extras"]))
+    if kwargs["req"]:
+        kwargs["req"].extras = set(kwargs["extras"])
+
     ireq = InstallRequirement(**kwargs)
 
     # If the original_link was None, keep it so. Passing `link` as an

tests/conftest.py

@@ -28,7 +28,7 @@
 from pip._vendor.packaging.version import Version
 from pip._vendor.pkg_resources import Requirement
 
-from piptools._compat import PIP_VERSION, Distribution
+from piptools._compat import Distribution
 from piptools.cache import DependencyCache
 from piptools.exceptions import NoCandidateFound
 from piptools.locations import DEFAULT_CONFIG_FILE_NAMES
@@ -37,6 +37,7 @@
 from piptools.repositories.base import BaseRepository
 from piptools.resolver import BacktrackingResolver, LegacyResolver
 from piptools.utils import (
+    PIP_VERSION,
     as_tuple,
     is_url_requirement,
     key_from_ireq,

tests/test_cli_compile.py

@@ -2375,6 +2375,49 @@ def test_combine_different_extras_of_the_same_package(
     )
 
 
+def test_canonicalize_extras(pip_conf, runner, tmp_path, make_package, make_wheel):
+    """
+    Ensure extras are written in a consistent format.
+    """
+    pkgs = [
+        make_package(
+            "fake-sqlalchemy",
+            version="0.1",
+            extras_require={"fake-postgresql_psycoPG2BINARY": ["fake-greenlet"]},
+        ),
+        make_package(
+            "fake-greenlet",
+            version="0.2",
+        ),
+    ]
+
+    dists_dir = tmp_path / "dists"
+    for pkg in pkgs:
+        make_wheel(pkg, dists_dir)
+
+    with open("requirements.in", "w") as req_in:
+        req_in.write("fake-sqlalchemy[FAKE_postgresql-psycopg2binary]\n")
+
+    out = runner.invoke(
+        cli,
+        [
+            "--output-file",
+            "-",
+            "--find-links",
+            str(dists_dir),
+            "--no-header",
+            "--no-emit-options",
+            "--no-annotate",
+            "--no-strip-extras",
+        ],
+    )
+    assert out.exit_code == 0
+    assert (
+        "fake-sqlalchemy[fake-postgresql-psycopg2binary]==0.1"
+        in out.stdout.splitlines()
+    )
+
+
 @pytest.mark.parametrize(
     ("pkg2_install_requires", "req_in_content", "out_expected_content"),
     (