From ae52918001bbee9e306227a7ebf7f6b8e3915d29 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com>
Date: Sun, 27 Dec 2020 11:41:55 +0000
Subject: [PATCH 01/16] Merge pull request #9289 from
 uranusjr/new-resolver-lazy-insert

---
 news/9203.bugfix.rst                          |  4 ++
 news/9246.bugfix.rst                          |  4 ++
 src/pip/_internal/exceptions.py               | 15 +++++
 .../resolution/resolvelib/candidates.py       | 23 ++-----
 .../resolution/resolvelib/factory.py          | 52 +++++++++++++---
 .../resolution/resolvelib/found_candidates.py | 36 +++++++----
 .../resolution/resolvelib/requirements.py     | 41 +++++++++++++
 src/pip/_internal/utils/subprocess.py         |  8 +--
 tests/functional/test_new_resolver.py         | 61 +++++++++++++++++++
 tests/unit/test_utils_subprocess.py           |  8 +--
 10 files changed, 205 insertions(+), 47 deletions(-)
 create mode 100644 news/9203.bugfix.rst
 create mode 100644 news/9246.bugfix.rst

diff --git a/news/9203.bugfix.rst b/news/9203.bugfix.rst
new file mode 100644
index 00000000000..38320218fbb
--- /dev/null
+++ b/news/9203.bugfix.rst
@@ -0,0 +1,4 @@
+New resolver: Discard a faulty distribution, instead of quitting outright.
+This implementation is taken from 20.2.2, with a fix that always makes the
+resolver iterate through candidates from indexes lazily, to avoid downloading
+candidates we do not need.
diff --git a/news/9246.bugfix.rst b/news/9246.bugfix.rst
new file mode 100644
index 00000000000..93f7f18f9f5
--- /dev/null
+++ b/news/9246.bugfix.rst
@@ -0,0 +1,4 @@
+New resolver: Discard a source distribution if it fails to generate metadata,
+instead of quitting outright. This implementation is taken from 20.2.2, with a
+fix that always makes the resolver iterate through candidates from indexes
+lazily, to avoid downloading candidates we do not need.
diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py
index 56482caf77b..3f2659e8792 100644
--- a/src/pip/_internal/exceptions.py
+++ b/src/pip/_internal/exceptions.py
@@ -151,6 +151,21 @@ def __str__(self):
         )
 
 
+class InstallationSubprocessError(InstallationError):
+    """A subprocess call failed during installation."""
+    def __init__(self, returncode, description):
+        # type: (int, str) -> None
+        self.returncode = returncode
+        self.description = description
+
+    def __str__(self):
+        # type: () -> str
+        return (
+            "Command errored out with exit status {}: {} "
+            "Check the logs for full command output."
+        ).format(self.returncode, self.description)
+
+
 class HashErrors(InstallationError):
     """Multiple HashError instances rolled into one for reporting"""
 
diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py
index cd1f188706f..83b6c98ab6a 100644
--- a/src/pip/_internal/resolution/resolvelib/candidates.py
+++ b/src/pip/_internal/resolution/resolvelib/candidates.py
@@ -141,7 +141,7 @@ def __init__(
         self._ireq = ireq
         self._name = name
         self._version = version
-        self._dist = None  # type: Optional[Distribution]
+        self.dist = self._prepare()
 
     def __str__(self):
         # type: () -> str
@@ -209,8 +209,6 @@ def _prepare_distribution(self):
     def _check_metadata_consistency(self, dist):
         # type: (Distribution) -> None
         """Check for consistency of project name and version of dist."""
-        # TODO: (Longer term) Rather than abort, reject this candidate
-        #       and backtrack. This would need resolvelib support.
         name = canonicalize_name(dist.project_name)
         if self._name is not None and self._name != name:
             raise MetadataInconsistent(self._ireq, "name", dist.project_name)
@@ -219,25 +217,17 @@ def _check_metadata_consistency(self, dist):
             raise MetadataInconsistent(self._ireq, "version", dist.version)
 
     def _prepare(self):
-        # type: () -> None
-        if self._dist is not None:
-            return
+        # type: () -> Distribution
         try:
             dist = self._prepare_distribution()
         except HashError as e:
+            # Provide HashError the underlying ireq that caused it. This
+            # provides context for the resulting error message to show the
+            # offending line to the user.
             e.req = self._ireq
             raise
-
-        assert dist is not None, "Distribution already installed"
         self._check_metadata_consistency(dist)
-        self._dist = dist
-
-    @property
-    def dist(self):
-        # type: () -> Distribution
-        if self._dist is None:
-            self._prepare()
-        return self._dist
+        return dist
 
     def _get_requires_python_dependency(self):
         # type: () -> Optional[Requirement]
@@ -261,7 +251,6 @@ def iter_dependencies(self, with_requires):
 
     def get_install_requirement(self):
         # type: () -> Optional[InstallRequirement]
-        self._prepare()
         return self._ireq
 
 
diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py
index b4c7bf11351..35345c5f0a1 100644
--- a/src/pip/_internal/resolution/resolvelib/factory.py
+++ b/src/pip/_internal/resolution/resolvelib/factory.py
@@ -5,6 +5,8 @@
 from pip._internal.exceptions import (
     DistributionNotFound,
     InstallationError,
+    InstallationSubprocessError,
+    MetadataInconsistent,
     UnsupportedPythonVersion,
     UnsupportedWheel,
 )
@@ -33,6 +35,7 @@
     ExplicitRequirement,
     RequiresPythonRequirement,
     SpecifierRequirement,
+    UnsatisfiableRequirement,
 )
 
 if MYPY_CHECK_RUNNING:
@@ -94,6 +97,7 @@ def __init__(
         self._force_reinstall = force_reinstall
         self._ignore_requires_python = ignore_requires_python
 
+        self._build_failures = {}  # type: Cache[InstallationError]
         self._link_candidate_cache = {}  # type: Cache[LinkCandidate]
         self._editable_candidate_cache = {}  # type: Cache[EditableCandidate]
         self._installed_candidate_cache = {
@@ -136,21 +140,40 @@ def _make_candidate_from_link(
         name,  # type: Optional[str]
         version,  # type: Optional[_BaseVersion]
     ):
-        # type: (...) -> Candidate
+        # type: (...) -> Optional[Candidate]
         # TODO: Check already installed candidate, and use it if the link and
         # editable flag match.
+
+        if link in self._build_failures:
+            # We already tried this candidate before, and it does not build.
+            # Don't bother trying again.
+            return None
+
         if template.editable:
             if link not in self._editable_candidate_cache:
-                self._editable_candidate_cache[link] = EditableCandidate(
-                    link, template, factory=self, name=name, version=version,
-                )
+                try:
+                    self._editable_candidate_cache[link] = EditableCandidate(
+                        link, template, factory=self,
+                        name=name, version=version,
+                    )
+                except (InstallationSubprocessError, MetadataInconsistent) as e:
+                    logger.warning("Discarding %s. %s", link, e)
+                    self._build_failures[link] = e
+                    return None
             base = self._editable_candidate_cache[link]  # type: BaseCandidate
         else:
             if link not in self._link_candidate_cache:
-                self._link_candidate_cache[link] = LinkCandidate(
-                    link, template, factory=self, name=name, version=version,
-                )
+                try:
+                    self._link_candidate_cache[link] = LinkCandidate(
+                        link, template, factory=self,
+                        name=name, version=version,
+                    )
+                except (InstallationSubprocessError, MetadataInconsistent) as e:
+                    logger.warning("Discarding %s. %s", link, e)
+                    self._build_failures[link] = e
+                    return None
             base = self._link_candidate_cache[link]
+
         if extras:
             return ExtrasCandidate(base, extras)
         return base
@@ -210,13 +233,16 @@ def iter_index_candidates():
             for ican in reversed(icans):
                 if not all_yanked and ican.link.is_yanked:
                     continue
-                yield self._make_candidate_from_link(
+                candidate = self._make_candidate_from_link(
                     link=ican.link,
                     extras=extras,
                     template=template,
                     name=name,
                     version=ican.version,
                 )
+                if candidate is None:
+                    continue
+                yield candidate
 
         return FoundCandidates(
             iter_index_candidates,
@@ -280,6 +306,16 @@ def make_requirement_from_install_req(self, ireq, requested_extras):
             name=canonicalize_name(ireq.name) if ireq.name else None,
             version=None,
         )
+        if cand is None:
+            # There's no way we can satisfy a URL requirement if the underlying
+            # candidate fails to build. An unnamed URL must be user-supplied, so
+            # we fail eagerly. If the URL is named, an unsatisfiable requirement
+            # can make the resolver do the right thing, either backtrack (and
+            # maybe find some other requirement that's buildable) or raise a
+            # ResolutionImpossible eventually.
+            if not ireq.name:
+                raise self._build_failures[ireq.link]
+            return UnsatisfiableRequirement(canonicalize_name(ireq.name))
         return self.make_requirement_from_candidate(cand)
 
     def make_requirement_from_candidate(self, candidate):
diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py
index a669e893670..a79a5cbda74 100644
--- a/src/pip/_internal/resolution/resolvelib/found_candidates.py
+++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py
@@ -1,5 +1,14 @@
+"""Utilities to lazily create and visit candidates found.
+
+Creating and visiting a candidate is a *very* costly operation. It involves
+fetching, extracting, potentially building modules from source, and verifying
+distribution metadata. It is therefore crucial for performance to keep
+everything here lazy all the way down, so we only touch candidates that we
+absolutely need, and not "download the world" when we only need one version of
+something.
+"""
+
 import itertools
-import operator
 
 from pip._vendor.six.moves import collections_abc  # type: ignore
 
@@ -32,18 +41,21 @@ def _insert_installed(installed, others):
     already-installed package. Candidates from index are returned in their
     normal ordering, except replaced when the version is already installed.
 
-    Since candidates from index are already sorted by reverse version order,
-    `sorted()` here would keep the ordering mostly intact, only shuffling the
-    already-installed candidate into the correct position. We put the already-
-    installed candidate in front of those from the index, so it's put in front
-    after sorting due to Python sorting's stableness guarentee.
+    The implementation iterates through and yields other candidates, inserting
+    the installed candidate exactly once before we start yielding older or
+    equivalent candidates, or after all other candidates if they are all newer.
     """
-    candidates = sorted(
-        itertools.chain([installed], others),
-        key=operator.attrgetter("version"),
-        reverse=True,
-    )
-    return iter(candidates)
+    installed_yielded = False
+    for candidate in others:
+        # If the installed candidate is better, yield it first.
+        if not installed_yielded and installed.version >= candidate.version:
+            yield installed
+            installed_yielded = True
+        yield candidate
+
+    # If the installed candidate is older than all other candidates.
+    if not installed_yielded:
+        yield installed
 
 
 class FoundCandidates(collections_abc.Sequence):
diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py
index d926d0a0656..1229f353750 100644
--- a/src/pip/_internal/resolution/resolvelib/requirements.py
+++ b/src/pip/_internal/resolution/resolvelib/requirements.py
@@ -158,3 +158,44 @@ def is_satisfied_by(self, candidate):
         # already implements the prerelease logic, and would have filtered out
         # prerelease candidates if the user does not expect them.
         return self.specifier.contains(candidate.version, prereleases=True)
+
+
+class UnsatisfiableRequirement(Requirement):
+    """A requirement that cannot be satisfied.
+    """
+    def __init__(self, name):
+        # type: (str) -> None
+        self._name = name
+
+    def __str__(self):
+        # type: () -> str
+        return "{} (unavailable)".format(self._name)
+
+    def __repr__(self):
+        # type: () -> str
+        return "{class_name}({name!r})".format(
+            class_name=self.__class__.__name__,
+            name=str(self._name),
+        )
+
+    @property
+    def project_name(self):
+        # type: () -> str
+        return self._name
+
+    @property
+    def name(self):
+        # type: () -> str
+        return self._name
+
+    def format_for_error(self):
+        # type: () -> str
+        return str(self)
+
+    def get_candidate_lookup(self):
+        # type: () -> CandidateLookup
+        return None, None
+
+    def is_satisfied_by(self, candidate):
+        # type: (Candidate) -> bool
+        return False
diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py
index 605e711e603..325897c8739 100644
--- a/src/pip/_internal/utils/subprocess.py
+++ b/src/pip/_internal/utils/subprocess.py
@@ -7,7 +7,7 @@
 from pip._vendor.six.moves import shlex_quote
 
 from pip._internal.cli.spinners import SpinnerInterface, open_spinner
-from pip._internal.exceptions import InstallationError
+from pip._internal.exceptions import InstallationSubprocessError
 from pip._internal.utils.compat import console_to_str, str_to_display
 from pip._internal.utils.logging import subprocess_logger
 from pip._internal.utils.misc import HiddenText, path_to_display
@@ -233,11 +233,7 @@ def call_subprocess(
                     exit_status=proc.returncode,
                 )
                 subprocess_logger.error(msg)
-            exc_msg = (
-                'Command errored out with exit status {}: {} '
-                'Check the logs for full command output.'
-            ).format(proc.returncode, command_desc)
-            raise InstallationError(exc_msg)
+            raise InstallationSubprocessError(proc.returncode, command_desc)
         elif on_returncode == 'warn':
             subprocess_logger.warning(
                 'Command "%s" had error code %s in %s',
diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py
index b730b3cbdf9..e2c76fe5638 100644
--- a/tests/functional/test_new_resolver.py
+++ b/tests/functional/test_new_resolver.py
@@ -1218,3 +1218,64 @@ def test_new_resolver_does_not_reinstall_when_from_a_local_index(script):
     assert "Installing collected packages: simple" not in result.stdout, str(result)
     assert "Requirement already satisfied: simple" in result.stdout, str(result)
     assert_installed(script, simple="0.1.0")
+
+
+def test_new_resolver_skip_inconsistent_metadata(script):
+    create_basic_wheel_for_package(script, "A", "1")
+
+    a_2 = create_basic_wheel_for_package(script, "A", "2")
+    a_2.rename(a_2.parent.joinpath("a-3-py2.py3-none-any.whl"))
+
+    result = script.pip(
+        "install",
+        "--no-cache-dir", "--no-index",
+        "--find-links", script.scratch_path,
+        "--verbose",
+        "A",
+        allow_stderr_warning=True,
+    )
+
+    assert " different version in metadata: '2'" in result.stderr, str(result)
+    assert_installed(script, a="1")
+
+
+@pytest.mark.parametrize(
+    "upgrade",
+    [True, False],
+    ids=["upgrade", "no-upgrade"],
+)
+def test_new_resolver_lazy_fetch_candidates(script, upgrade):
+    create_basic_wheel_for_package(script, "myuberpkg", "1")
+    create_basic_wheel_for_package(script, "myuberpkg", "2")
+    create_basic_wheel_for_package(script, "myuberpkg", "3")
+
+    # Install an old version first.
+    script.pip(
+        "install",
+        "--no-cache-dir", "--no-index",
+        "--find-links", script.scratch_path,
+        "myuberpkg==1",
+    )
+
+    # Now install the same package again, maybe with the upgrade flag.
+    if upgrade:
+        pip_upgrade_args = ["--upgrade"]
+    else:
+        pip_upgrade_args = []
+    result = script.pip(
+        "install",
+        "--no-cache-dir", "--no-index",
+        "--find-links", script.scratch_path,
+        "myuberpkg",
+        *pip_upgrade_args  # Trailing comma fails on Python 2.
+    )
+
+    # pip should install the version preferred by the strategy...
+    if upgrade:
+        assert_installed(script, myuberpkg="3")
+    else:
+        assert_installed(script, myuberpkg="1")
+
+    # But should reach there in the best route possible, without trying
+    # candidates it does not need to.
+    assert "myuberpkg-2" not in result.stdout, str(result)
diff --git a/tests/unit/test_utils_subprocess.py b/tests/unit/test_utils_subprocess.py
index b0de2bf578d..1a031065114 100644
--- a/tests/unit/test_utils_subprocess.py
+++ b/tests/unit/test_utils_subprocess.py
@@ -7,7 +7,7 @@
 import pytest
 
 from pip._internal.cli.spinners import SpinnerInterface
-from pip._internal.exceptions import InstallationError
+from pip._internal.exceptions import InstallationSubprocessError
 from pip._internal.utils.misc import hide_value
 from pip._internal.utils.subprocess import (
     call_subprocess,
@@ -276,7 +276,7 @@ def test_info_logging__subprocess_error(self, capfd, caplog):
         command = 'print("Hello"); print("world"); exit("fail")'
         args, spinner = self.prepare_call(caplog, log_level, command=command)
 
-        with pytest.raises(InstallationError) as exc:
+        with pytest.raises(InstallationSubprocessError) as exc:
             call_subprocess(args, spinner=spinner)
         result = None
         exc_message = str(exc.value)
@@ -360,7 +360,7 @@ def test_info_logging_with_show_stdout_true(self, capfd, caplog):
             # log level is only WARNING.
             (0, True, None, WARNING, (None, 'done', 2)),
             # Test a non-zero exit status.
-            (3, False, None, INFO, (InstallationError, 'error', 2)),
+            (3, False, None, INFO, (InstallationSubprocessError, 'error', 2)),
             # Test a non-zero exit status also in extra_ok_returncodes.
             (3, False, (3, ), INFO, (None, 'done', 2)),
     ])
@@ -396,7 +396,7 @@ def test_spinner_finish(
         assert spinner.spin_count == expected_spin_count
 
     def test_closes_stdin(self):
-        with pytest.raises(InstallationError):
+        with pytest.raises(InstallationSubprocessError):
             call_subprocess(
                 [sys.executable, '-c', 'input()'],
                 show_stdout=True,

From e4edf1f652bd5cd5af343383f8e0746642b03bb7 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com>
Date: Sun, 27 Dec 2020 11:59:44 +0000
Subject: [PATCH 02/16] Merge pull request #9315 from
 pradyunsg/better-search-errors

---
 news/9315.feature.rst                | 1 +
 src/pip/_internal/commands/search.py | 9 ++++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 news/9315.feature.rst

diff --git a/news/9315.feature.rst b/news/9315.feature.rst
new file mode 100644
index 00000000000..64d1f25338b
--- /dev/null
+++ b/news/9315.feature.rst
@@ -0,0 +1 @@
+Improve presentation of XMLRPC errors in pip search.
diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py
index 146d653e55f..185495e7688 100644
--- a/src/pip/_internal/commands/search.py
+++ b/src/pip/_internal/commands/search.py
@@ -79,7 +79,14 @@ def search(self, query, options):
 
         transport = PipXmlrpcTransport(index_url, session)
         pypi = xmlrpc_client.ServerProxy(index_url, transport)
-        hits = pypi.search({'name': query, 'summary': query}, 'or')
+        try:
+            hits = pypi.search({'name': query, 'summary': query}, 'or')
+        except xmlrpc_client.Fault as fault:
+            message = "XMLRPC request failed [code: {code}]\n{string}".format(
+                code=fault.faultCode,
+                string=fault.faultString,
+            )
+            raise CommandError(message)
         return hits
 
 

From 084463b7700b6115519f06cd73f8d8cc40a0e35a Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com>
Date: Sun, 27 Dec 2020 15:26:43 +0000
Subject: [PATCH 03/16] Merge pull request #9369 from uranusjr/resolvelib-054

---
 news/9180.bugfix.rst                    |  1 +
 news/resolvelib.vendor.rst              |  1 +
 src/pip/_vendor/resolvelib/__init__.py  |  2 +-
 src/pip/_vendor/resolvelib/resolvers.py | 36 +++++++++++++++----------
 src/pip/_vendor/vendor.txt              |  2 +-
 5 files changed, 26 insertions(+), 16 deletions(-)
 create mode 100644 news/9180.bugfix.rst
 create mode 100644 news/resolvelib.vendor.rst

diff --git a/news/9180.bugfix.rst b/news/9180.bugfix.rst
new file mode 100644
index 00000000000..e597c1ad90a
--- /dev/null
+++ b/news/9180.bugfix.rst
@@ -0,0 +1 @@
+Fix error when an existing incompatibility is unable to be applied to a backtracked state.
diff --git a/news/resolvelib.vendor.rst b/news/resolvelib.vendor.rst
new file mode 100644
index 00000000000..680da3be1e7
--- /dev/null
+++ b/news/resolvelib.vendor.rst
@@ -0,0 +1 @@
+Upgrade resolvelib to 0.5.4.
diff --git a/src/pip/_vendor/resolvelib/__init__.py b/src/pip/_vendor/resolvelib/__init__.py
index 5a400f23ed1..f023ad63154 100644
--- a/src/pip/_vendor/resolvelib/__init__.py
+++ b/src/pip/_vendor/resolvelib/__init__.py
@@ -11,7 +11,7 @@
     "ResolutionTooDeep",
 ]
 
-__version__ = "0.5.3"
+__version__ = "0.5.4"
 
 
 from .providers import AbstractProvider, AbstractResolver
diff --git a/src/pip/_vendor/resolvelib/resolvers.py b/src/pip/_vendor/resolvelib/resolvers.py
index acf0f8a6b43..bb88d8c2c75 100644
--- a/src/pip/_vendor/resolvelib/resolvers.py
+++ b/src/pip/_vendor/resolvelib/resolvers.py
@@ -257,7 +257,7 @@ def _backtrack(self):
             information from Y to Y'.
         4a. If this causes Y' to conflict, we need to backtrack again. Make Y'
             the new Z and go back to step 2.
-        4b. If the incompatibilites apply cleanly, end backtracking.
+        4b. If the incompatibilities apply cleanly, end backtracking.
         """
         while len(self._states) >= 3:
             # Remove the state that triggered backtracking.
@@ -271,28 +271,36 @@ def _backtrack(self):
                 for k, v in broken_state.criteria.items()
             ]
 
+            # Also mark the newly known incompatibility.
+            incompatibilities_from_broken.append((name, [candidate]))
+
             self._r.backtracking(candidate)
 
             # Create a new state from the last known-to-work one, and apply
             # the previously gathered incompatibility information.
-            self._push_new_state()
-            for k, incompatibilities in incompatibilities_from_broken:
-                try:
-                    crit = self.state.criteria[k]
-                except KeyError:
-                    continue
-                self.state.criteria[k] = crit.excluded_of(incompatibilities)
+            def _patch_criteria():
+                for k, incompatibilities in incompatibilities_from_broken:
+                    if not incompatibilities:
+                        continue
+                    try:
+                        criterion = self.state.criteria[k]
+                    except KeyError:
+                        continue
+                    criterion = criterion.excluded_of(incompatibilities)
+                    if criterion is None:
+                        return False
+                    self.state.criteria[k] = criterion
+                return True
 
-            # Mark the newly known incompatibility.
-            criterion = self.state.criteria[name].excluded_of([candidate])
+            self._push_new_state()
+            success = _patch_criteria()
 
             # It works! Let's work on this new state.
-            if criterion:
-                self.state.criteria[name] = criterion
+            if success:
                 return True
 
-            # State does not work after adding the new incompatibility
-            # information. Try the still previous state.
+            # State does not work after applying known incompatibilities.
+            # Try the still previous state.
 
         # No way to backtrack anymore.
         return False
diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt
index 15c000339ae..712fb77d46b 100644
--- a/src/pip/_vendor/vendor.txt
+++ b/src/pip/_vendor/vendor.txt
@@ -16,7 +16,7 @@ requests==2.25.0
     chardet==3.0.4
     idna==2.10
     urllib3==1.26.2
-resolvelib==0.5.3
+resolvelib==0.5.4
 retrying==1.3.3
 setuptools==44.0.0
 six==1.15.0

From f1a9ef70e8b825347fcc6ce0d31f23cc6412efb0 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com>
Date: Tue, 5 Jan 2021 21:47:53 +0000
Subject: [PATCH 04/16] Merge pull request #9320 from
 uranusjr/wheel-check-valid

Verify built wheel contains valid metadata
---
 news/9206.feature.rst                 |  3 ++
 src/pip/_internal/commands/install.py |  1 +
 src/pip/_internal/commands/wheel.py   |  9 ++++
 src/pip/_internal/wheel_builder.py    | 60 ++++++++++++++++++++++++++-
 4 files changed, 71 insertions(+), 2 deletions(-)
 create mode 100644 news/9206.feature.rst

diff --git a/news/9206.feature.rst b/news/9206.feature.rst
new file mode 100644
index 00000000000..90cd2cf99fb
--- /dev/null
+++ b/news/9206.feature.rst
@@ -0,0 +1,3 @@
+``pip wheel`` now verifies the built wheel contains valid metadata, and can be
+installed by a subsequent ``pip install``. This can be disabled with
+``--no-verify``.
diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py
index a4e10f260a2..0f6c384e5cd 100644
--- a/src/pip/_internal/commands/install.py
+++ b/src/pip/_internal/commands/install.py
@@ -347,6 +347,7 @@ def run(self, options, args):
             _, build_failures = build(
                 reqs_to_build,
                 wheel_cache=wheel_cache,
+                verify=True,
                 build_options=[],
                 global_options=[],
             )
diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py
index 39fd2bf8128..a9f66258a14 100644
--- a/src/pip/_internal/commands/wheel.py
+++ b/src/pip/_internal/commands/wheel.py
@@ -81,6 +81,14 @@ def add_options(self):
         self.cmd_opts.add_option(cmdoptions.build_dir())
         self.cmd_opts.add_option(cmdoptions.progress_bar())
 
+        self.cmd_opts.add_option(
+            '--no-verify',
+            dest='no_verify',
+            action='store_true',
+            default=False,
+            help="Don't verify if built wheel is valid.",
+        )
+
         self.cmd_opts.add_option(
             '--global-option',
             dest='global_options',
@@ -166,6 +174,7 @@ def run(self, options, args):
         build_successes, build_failures = build(
             reqs_to_build,
             wheel_cache=wheel_cache,
+            verify=(not options.no_verify),
             build_options=options.build_options or [],
             global_options=options.global_options or [],
         )
diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py
index 27fce66c264..dbc34d0952b 100644
--- a/src/pip/_internal/wheel_builder.py
+++ b/src/pip/_internal/wheel_builder.py
@@ -5,8 +5,15 @@
 import os.path
 import re
 import shutil
+import zipfile
 
+from pip._vendor.packaging.utils import canonicalize_name, canonicalize_version
+from pip._vendor.packaging.version import InvalidVersion, Version
+from pip._vendor.pkg_resources import Distribution
+
+from pip._internal.exceptions import InvalidWheelFilename, UnsupportedWheel
 from pip._internal.models.link import Link
+from pip._internal.models.wheel import Wheel
 from pip._internal.operations.build.wheel import build_wheel_pep517
 from pip._internal.operations.build.wheel_legacy import build_wheel_legacy
 from pip._internal.utils.logging import indent_log
@@ -16,6 +23,7 @@
 from pip._internal.utils.temp_dir import TempDirectory
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.urls import path_to_url
+from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
 from pip._internal.vcs import vcs
 
 if MYPY_CHECK_RUNNING:
@@ -160,9 +168,49 @@ def _always_true(_):
     return True
 
 
+def _get_metadata_version(dist):
+    # type: (Distribution) -> Optional[Version]
+    for line in dist.get_metadata_lines(dist.PKG_INFO):
+        if line.lower().startswith("metadata-version:"):
+            value = line.split(":", 1)[-1].strip()
+            try:
+                return Version(value)
+            except InvalidVersion:
+                msg = "Invalid Metadata-Version: {}".format(value)
+                raise UnsupportedWheel(msg)
+    raise UnsupportedWheel("Missing Metadata-Version")
+
+
+def _verify_one(req, wheel_path):
+    # type: (InstallRequirement, str) -> None
+    canonical_name = canonicalize_name(req.name)
+    w = Wheel(os.path.basename(wheel_path))
+    if canonicalize_name(w.name) != canonical_name:
+        raise InvalidWheelFilename(
+            "Wheel has unexpected file name: expected {!r}, "
+            "got {!r}".format(canonical_name, w.name),
+        )
+    with zipfile.ZipFile(wheel_path, allowZip64=True) as zf:
+        dist = pkg_resources_distribution_for_wheel(
+            zf, canonical_name, wheel_path,
+        )
+    if canonicalize_version(dist.version) != canonicalize_version(w.version):
+        raise InvalidWheelFilename(
+            "Wheel has unexpected file name: expected {!r}, "
+            "got {!r}".format(dist.version, w.version),
+        )
+    if (_get_metadata_version(dist) >= Version("1.2")
+            and not isinstance(dist.parsed_version, Version)):
+        raise UnsupportedWheel(
+            "Metadata 1.2 mandates PEP 440 version, "
+            "but {!r} is not".format(dist.version)
+        )
+
+
 def _build_one(
     req,  # type: InstallRequirement
     output_dir,  # type: str
+    verify,  # type: bool
     build_options,  # type: List[str]
     global_options,  # type: List[str]
 ):
@@ -182,9 +230,16 @@ def _build_one(
 
     # Install build deps into temporary directory (PEP 518)
     with req.build_env:
-        return _build_one_inside_env(
+        wheel_path = _build_one_inside_env(
             req, output_dir, build_options, global_options
         )
+    if wheel_path and verify:
+        try:
+            _verify_one(req, wheel_path)
+        except (InvalidWheelFilename, UnsupportedWheel) as e:
+            logger.warning("Built wheel for %s is invalid: %s", req.name, e)
+            return None
+    return wheel_path
 
 
 def _build_one_inside_env(
@@ -257,6 +312,7 @@ def _clean_one_legacy(req, global_options):
 def build(
     requirements,  # type: Iterable[InstallRequirement]
     wheel_cache,  # type: WheelCache
+    verify,  # type: bool
     build_options,  # type: List[str]
     global_options,  # type: List[str]
 ):
@@ -280,7 +336,7 @@ def build(
         for req in requirements:
             cache_dir = _get_cache_dir(req, wheel_cache)
             wheel_file = _build_one(
-                req, cache_dir, build_options, global_options
+                req, cache_dir, verify, build_options, global_options
             )
             if wheel_file:
                 # Update the link for this.

From 49f972210027f5bc013e6104fdfa0acf11df03b3 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com>
Date: Mon, 11 Jan 2021 19:31:03 +0000
Subject: [PATCH 05/16] Merge pull request #9432 from
 uranusjr/new-resolver-dedup-on-backtracking

Avoid downloading multiple candidates of a same version
---
 .../resolution/resolvelib/factory.py          |  4 +++
 .../resolution/resolvelib/found_candidates.py | 29 +++++++------------
 2 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py
index 35345c5f0a1..e81595b8c90 100644
--- a/src/pip/_internal/resolution/resolvelib/factory.py
+++ b/src/pip/_internal/resolution/resolvelib/factory.py
@@ -230,9 +230,12 @@ def iter_index_candidates():
             all_yanked = all(ican.link.is_yanked for ican in icans)
 
             # PackageFinder returns earlier versions first, so we reverse.
+            versions_found = set()  # type: Set[_BaseVersion]
             for ican in reversed(icans):
                 if not all_yanked and ican.link.is_yanked:
                     continue
+                if ican.version in versions_found:
+                    continue
                 candidate = self._make_candidate_from_link(
                     link=ican.link,
                     extras=extras,
@@ -243,6 +246,7 @@ def iter_index_candidates():
                 if candidate is None:
                     continue
                 yield candidate
+                versions_found.add(ican.version)
 
         return FoundCandidates(
             iter_index_candidates,
diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py
index a79a5cbda74..50359b64fee 100644
--- a/src/pip/_internal/resolution/resolvelib/found_candidates.py
+++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py
@@ -16,23 +16,11 @@
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 if MYPY_CHECK_RUNNING:
-    from typing import Callable, Iterator, Optional, Set
-
-    from pip._vendor.packaging.version import _BaseVersion
+    from typing import Callable, Iterator, Optional
 
     from .base import Candidate
 
 
-def _deduplicated_by_version(candidates):
-    # type: (Iterator[Candidate]) -> Iterator[Candidate]
-    returned = set()  # type: Set[_BaseVersion]
-    for candidate in candidates:
-        if candidate.version in returned:
-            continue
-        returned.add(candidate.version)
-        yield candidate
-
-
 def _insert_installed(installed, others):
     # type: (Candidate, Iterator[Candidate]) -> Iterator[Candidate]
     """Iterator for ``FoundCandidates``.
@@ -86,12 +74,15 @@ def __getitem__(self, index):
     def __iter__(self):
         # type: () -> Iterator[Candidate]
         if not self._installed:
-            candidates = self._get_others()
-        elif self._prefers_installed:
-            candidates = itertools.chain([self._installed], self._get_others())
-        else:
-            candidates = _insert_installed(self._installed, self._get_others())
-        return _deduplicated_by_version(candidates)
+            return self._get_others()
+        others = (
+            candidate
+            for candidate in self._get_others()
+            if candidate.version != self._installed.version
+        )
+        if self._prefers_installed:
+            return itertools.chain([self._installed], others)
+        return _insert_installed(self._installed, others)
 
     def __len__(self):
         # type: () -> int

From 139aaf82030ef4725f2a362fdd890503e684b171 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Mon, 21 Dec 2020 12:38:41 +0100
Subject: [PATCH 06/16] Revert "Remove on_returncode parameter from
 call_subprocess"

This reverts commit ab3ee7191ca47294f8827916180969e23f5e0381.
---
 src/pip/_internal/vcs/git.py            | 10 ++----
 src/pip/_internal/vcs/mercurial.py      |  1 +
 src/pip/_internal/vcs/versioncontrol.py | 42 ++++++++++++++++---------
 3 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index 1831aede58a..141263803b9 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -135,13 +135,8 @@ def get_revision_sha(cls, dest, rev):
           rev: the revision name.
         """
         # Pass rev to pre-filter the list.
-
-        output = ''
-        try:
-            output = cls.run_command(['show-ref', rev], cwd=dest)
-        except SubProcessError:
-            pass
-
+        output = cls.run_command(['show-ref', rev], cwd=dest,
+                                 on_returncode='ignore')
         refs = {}
         for line in output.strip().splitlines():
             try:
@@ -420,6 +415,7 @@ def get_repository_root(cls, location):
             r = cls.run_command(
                 ['rev-parse', '--show-toplevel'],
                 cwd=location,
+                on_returncode='raise',
                 log_failed_cmd=False,
             )
         except BadCommand:
diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py
index 69763feaea4..b7f8073fd38 100644
--- a/src/pip/_internal/vcs/mercurial.py
+++ b/src/pip/_internal/vcs/mercurial.py
@@ -144,6 +144,7 @@ def get_repository_root(cls, location):
             r = cls.run_command(
                 ['root'],
                 cwd=location,
+                on_returncode='raise',
                 log_failed_cmd=False,
             )
         except BadCommand:
diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py
index 6724dcc697d..5b93b66091f 100644
--- a/src/pip/_internal/vcs/versioncontrol.py
+++ b/src/pip/_internal/vcs/versioncontrol.py
@@ -90,6 +90,7 @@ def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None):
 def call_subprocess(
     cmd,  # type: Union[List[str], CommandArgs]
     cwd=None,  # type: Optional[str]
+    on_returncode='raise',  # type: str
     extra_environ=None,  # type: Optional[Mapping[str, Any]]
     extra_ok_returncodes=None,  # type: Optional[Iterable[int]]
     log_failed_cmd=True  # type: Optional[bool]
@@ -157,21 +158,32 @@ def call_subprocess(
         proc.returncode and proc.returncode not in extra_ok_returncodes
     )
     if proc_had_error:
-        if not showing_subprocess and log_failed_cmd:
-            # Then the subprocess streams haven't been logged to the
-            # console yet.
-            msg = make_subprocess_output_error(
-                cmd_args=cmd,
-                cwd=cwd,
-                lines=all_output,
-                exit_status=proc.returncode,
+        if on_returncode == 'raise':
+            if not showing_subprocess and log_failed_cmd:
+                # Then the subprocess streams haven't been logged to the
+                # console yet.
+                msg = make_subprocess_output_error(
+                    cmd_args=cmd,
+                    cwd=cwd,
+                    lines=all_output,
+                    exit_status=proc.returncode,
+                )
+                subprocess_logger.error(msg)
+            exc_msg = (
+                'Command errored out with exit status {}: {} '
+                'Check the logs for full command output.'
+            ).format(proc.returncode, command_desc)
+            raise SubProcessError(exc_msg)
+        elif on_returncode == 'warn':
+            subprocess_logger.warning(
+                'Command "{}" had error code {} in {}'.format(
+                    command_desc, proc.returncode, cwd)
             )
-            subprocess_logger.error(msg)
-        exc_msg = (
-            'Command errored out with exit status {}: {} '
-            'Check the logs for full command output.'
-        ).format(proc.returncode, command_desc)
-        raise SubProcessError(exc_msg)
+        elif on_returncode == 'ignore':
+            pass
+        else:
+            raise ValueError('Invalid value: on_returncode={!r}'.format(
+                             on_returncode))
     return ''.join(all_output)
 
 
@@ -764,6 +776,7 @@ def run_command(
         cls,
         cmd,  # type: Union[List[str], CommandArgs]
         cwd=None,  # type: Optional[str]
+        on_returncode='raise',  # type: str
         extra_environ=None,  # type: Optional[Mapping[str, Any]]
         extra_ok_returncodes=None,  # type: Optional[Iterable[int]]
         log_failed_cmd=True  # type: bool
@@ -777,6 +790,7 @@ def run_command(
         cmd = make_command(cls.name, *cmd)
         try:
             return call_subprocess(cmd, cwd,
+                                   on_returncode=on_returncode,
                                    extra_environ=extra_environ,
                                    extra_ok_returncodes=extra_ok_returncodes,
                                    log_failed_cmd=log_failed_cmd)

From 5a4b665fbe88f736330e3db4f8f5f5950e1cf592 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Mon, 21 Dec 2020 12:44:23 +0100
Subject: [PATCH 07/16] Revert "Remove show_stdout from run_command args"

This reverts commit 94882fd1ed9171ea5a2f4b8904dbd8763f05ba68.
---
 src/pip/_internal/vcs/bazaar.py         |  7 ++--
 src/pip/_internal/vcs/git.py            | 19 +++++-----
 src/pip/_internal/vcs/mercurial.py      | 16 ++++----
 src/pip/_internal/vcs/subversion.py     |  9 +++--
 src/pip/_internal/vcs/versioncontrol.py | 50 +++++++++++++++++--------
 5 files changed, 62 insertions(+), 39 deletions(-)

diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py
index 3180713f7db..efe524492af 100644
--- a/src/pip/_internal/vcs/bazaar.py
+++ b/src/pip/_internal/vcs/bazaar.py
@@ -55,7 +55,8 @@ def export(self, location, url):
 
         url, rev_options = self.get_url_rev_options(url)
         self.run_command(
-            make_command('export', location, url, rev_options.to_args())
+            make_command('export', location, url, rev_options.to_args()),
+            show_stdout=False,
         )
 
     def fetch_new(self, dest, url, rev_options):
@@ -92,7 +93,7 @@ def get_url_rev_and_auth(cls, url):
 
     @classmethod
     def get_remote_url(cls, location):
-        urls = cls.run_command(['info'], cwd=location)
+        urls = cls.run_command(['info'], show_stdout=False, cwd=location)
         for line in urls.splitlines():
             line = line.strip()
             for x in ('checkout of branch: ',
@@ -107,7 +108,7 @@ def get_remote_url(cls, location):
     @classmethod
     def get_revision(cls, location):
         revision = cls.run_command(
-            ['revno'], cwd=location,
+            ['revno'], show_stdout=False, cwd=location,
         )
         return revision.splitlines()[-1]
 
diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index 141263803b9..a6d1396fc77 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -11,7 +11,7 @@
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 from pip._vendor.six.moves.urllib import request as urllib_request
 
-from pip._internal.exceptions import BadCommand, SubProcessError
+from pip._internal.exceptions import BadCommand, InstallationError, SubProcessError
 from pip._internal.utils.misc import display_path, hide_url
 from pip._internal.utils.subprocess import make_command
 from pip._internal.utils.temp_dir import TempDirectory
@@ -79,7 +79,7 @@ def is_immutable_rev_checkout(self, url, dest):
 
     def get_git_version(self):
         VERSION_PFX = 'git version '
-        version = self.run_command(['version'])
+        version = self.run_command(['version'], show_stdout=False)
         if version.startswith(VERSION_PFX):
             version = version[len(VERSION_PFX):].split()[0]
         else:
@@ -102,7 +102,7 @@ def get_current_branch(cls, location):
         # and to suppress the message to stderr.
         args = ['symbolic-ref', '-q', 'HEAD']
         output = cls.run_command(
-            args, extra_ok_returncodes=(1, ), cwd=location,
+            args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location,
         )
         ref = output.strip()
 
@@ -121,7 +121,7 @@ def export(self, location, url):
             self.unpack(temp_dir.path, url=url)
             self.run_command(
                 ['checkout-index', '-a', '-f', '--prefix', location],
-                cwd=temp_dir.path
+                show_stdout=False, cwd=temp_dir.path
             )
 
     @classmethod
@@ -136,7 +136,7 @@ def get_revision_sha(cls, dest, rev):
         """
         # Pass rev to pre-filter the list.
         output = cls.run_command(['show-ref', rev], cwd=dest,
-                                 on_returncode='ignore')
+                                 show_stdout=False, on_returncode='ignore')
         refs = {}
         for line in output.strip().splitlines():
             try:
@@ -310,7 +310,7 @@ def get_remote_url(cls, location):
         # exits with return code 1 if there are no matching lines.
         stdout = cls.run_command(
             ['config', '--get-regexp', r'remote\..*\.url'],
-            extra_ok_returncodes=(1, ), cwd=location,
+            extra_ok_returncodes=(1, ), show_stdout=False, cwd=location,
         )
         remotes = stdout.splitlines()
         try:
@@ -344,7 +344,7 @@ def get_revision(cls, location, rev=None):
         if rev is None:
             rev = 'HEAD'
         current_rev = cls.run_command(
-            ['rev-parse', rev], cwd=location,
+            ['rev-parse', rev], show_stdout=False, cwd=location,
         )
         return current_rev.strip()
 
@@ -357,7 +357,7 @@ def get_subdirectory(cls, location):
         # find the repo root
         git_dir = cls.run_command(
             ['rev-parse', '--git-dir'],
-            cwd=location).strip()
+            show_stdout=False, cwd=location).strip()
         if not os.path.isabs(git_dir):
             git_dir = os.path.join(location, git_dir)
         repo_root = os.path.abspath(os.path.join(git_dir, '..'))
@@ -415,6 +415,7 @@ def get_repository_root(cls, location):
             r = cls.run_command(
                 ['rev-parse', '--show-toplevel'],
                 cwd=location,
+                show_stdout=False,
                 on_returncode='raise',
                 log_failed_cmd=False,
             )
@@ -422,7 +423,7 @@ def get_repository_root(cls, location):
             logger.debug("could not determine if %s is under git control "
                          "because git is not available", location)
             return None
-        except SubProcessError:
+        except InstallationError:
             return None
         return os.path.normpath(r.rstrip('\r\n'))
 
diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py
index b7f8073fd38..75e903cc8a6 100644
--- a/src/pip/_internal/vcs/mercurial.py
+++ b/src/pip/_internal/vcs/mercurial.py
@@ -8,7 +8,7 @@
 
 from pip._vendor.six.moves import configparser
 
-from pip._internal.exceptions import BadCommand, SubProcessError
+from pip._internal.exceptions import BadCommand, InstallationError
 from pip._internal.utils.misc import display_path
 from pip._internal.utils.subprocess import make_command
 from pip._internal.utils.temp_dir import TempDirectory
@@ -47,7 +47,7 @@ def export(self, location, url):
             self.unpack(temp_dir.path, url=url)
 
             self.run_command(
-                ['archive', location], cwd=temp_dir.path
+                ['archive', location], show_stdout=False, cwd=temp_dir.path
             )
 
     def fetch_new(self, dest, url, rev_options):
@@ -92,7 +92,7 @@ def update(self, dest, url, rev_options):
     def get_remote_url(cls, location):
         url = cls.run_command(
             ['showconfig', 'paths.default'],
-            cwd=location).strip()
+            show_stdout=False, cwd=location).strip()
         if cls._is_local_repository(url):
             url = path_to_url(url)
         return url.strip()
@@ -103,7 +103,8 @@ def get_revision(cls, location):
         Return the repository-local changeset revision number, as an integer.
         """
         current_revision = cls.run_command(
-            ['parents', '--template={rev}'], cwd=location).strip()
+            ['parents', '--template={rev}'],
+            show_stdout=False, cwd=location).strip()
         return current_revision
 
     @classmethod
@@ -114,7 +115,7 @@ def get_requirement_revision(cls, location):
         """
         current_rev_hash = cls.run_command(
             ['parents', '--template={node}'],
-            cwd=location).strip()
+            show_stdout=False, cwd=location).strip()
         return current_rev_hash
 
     @classmethod
@@ -130,7 +131,7 @@ def get_subdirectory(cls, location):
         """
         # find the repo root
         repo_root = cls.run_command(
-            ['root'], cwd=location).strip()
+            ['root'], show_stdout=False, cwd=location).strip()
         if not os.path.isabs(repo_root):
             repo_root = os.path.abspath(os.path.join(location, repo_root))
         return find_path_to_setup_from_repo_root(location, repo_root)
@@ -144,6 +145,7 @@ def get_repository_root(cls, location):
             r = cls.run_command(
                 ['root'],
                 cwd=location,
+                show_stdout=False,
                 on_returncode='raise',
                 log_failed_cmd=False,
             )
@@ -151,7 +153,7 @@ def get_repository_root(cls, location):
             logger.debug("could not determine if %s is under hg control "
                          "because hg is not available", location)
             return None
-        except SubProcessError:
+        except InstallationError:
             return None
         return os.path.normpath(r.rstrip('\r\n'))
 
diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py
index eae09c19610..a57e19751fa 100644
--- a/src/pip/_internal/vcs/subversion.py
+++ b/src/pip/_internal/vcs/subversion.py
@@ -133,7 +133,7 @@ def get_remote_url(cls, location):
 
     @classmethod
     def _get_svn_url_rev(cls, location):
-        from pip._internal.exceptions import SubProcessError
+        from pip._internal.exceptions import InstallationError
 
         entries_path = os.path.join(location, cls.dirname, 'entries')
         if os.path.exists(entries_path):
@@ -166,12 +166,13 @@ def _get_svn_url_rev(cls, location):
                 # are only potentially needed for remote server requests.
                 xml = cls.run_command(
                     ['info', '--xml', location],
+                    show_stdout=False,
                 )
                 url = _svn_info_xml_url_re.search(xml).group(1)
                 revs = [
                     int(m.group(1)) for m in _svn_info_xml_rev_re.finditer(xml)
                 ]
-            except SubProcessError:
+            except InstallationError:
                 url, revs = None, []
 
         if revs:
@@ -217,7 +218,7 @@ def call_vcs_version(self):
         #   svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0)
         #      compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2
         version_prefix = 'svn, version '
-        version = self.run_command(['--version'])
+        version = self.run_command(['--version'], show_stdout=True)
 
         if not version.startswith(version_prefix):
             return ()
@@ -300,7 +301,7 @@ def export(self, location, url):
                 'export', self.get_remote_call_options(),
                 rev_options.to_args(), url, location,
             )
-            self.run_command(cmd_args)
+            self.run_command(cmd_args, show_stdout=False)
 
     def fetch_new(self, dest, url, rev_options):
         # type: (str, HiddenText, RevOptions) -> None
diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py
index 5b93b66091f..a4995c88f78 100644
--- a/src/pip/_internal/vcs/versioncontrol.py
+++ b/src/pip/_internal/vcs/versioncontrol.py
@@ -34,12 +34,10 @@
 
 if MYPY_CHECK_RUNNING:
     from typing import (
-        Any,
         Dict,
         Iterable,
         Iterator,
         List,
-        Mapping,
         Optional,
         Text,
         Tuple,
@@ -89,15 +87,17 @@ def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None):
 
 def call_subprocess(
     cmd,  # type: Union[List[str], CommandArgs]
+    show_stdout=False,  # type: bool
     cwd=None,  # type: Optional[str]
     on_returncode='raise',  # type: str
-    extra_environ=None,  # type: Optional[Mapping[str, Any]]
     extra_ok_returncodes=None,  # type: Optional[Iterable[int]]
     log_failed_cmd=True  # type: Optional[bool]
 ):
     # type: (...) -> Text
     """
     Args:
+      show_stdout: if true, use INFO to log the subprocess's stderr and
+        stdout streams.  Otherwise, use DEBUG.  Defaults to False.
       extra_ok_returncodes: an iterable of integer return codes that are
         acceptable, in addition to 0. Defaults to None, which means [].
       log_failed_cmd: if false, failed commands are not logged,
@@ -105,16 +105,33 @@ def call_subprocess(
     """
     if extra_ok_returncodes is None:
         extra_ok_returncodes = []
-
-    # log the subprocess output at DEBUG level.
-    log_subprocess = subprocess_logger.debug
-
-    env = os.environ.copy()
-    if extra_environ:
-        env.update(extra_environ)
+    # Most places in pip use show_stdout=False.
+    # What this means is--
+    #
+    # - We log this output of stdout and stderr at DEBUG level
+    # as it is received.
+    # - If DEBUG logging isn't enabled (e.g. if --verbose logging wasn't
+    #   requested), then we show a spinner so the user can still see the
+    #   subprocess is in progress.
+    # - If the subprocess exits with an error, we log the output to stderr
+    #   at ERROR level if it hasn't already been displayed to the console
+    #   (e.g. if --verbose logging wasn't enabled).  This way we don't log
+    #   the output to the console twice.
+    #
+    # If show_stdout=True, then the above is still done, but with DEBUG
+    # replaced by INFO.
+    if show_stdout:
+        # Then log the subprocess output at INFO level.
+        log_subprocess = subprocess_logger.info
+        used_level = logging.INFO
+    else:
+        # Then log the subprocess output using DEBUG.  This also ensures
+        # it will be logged to the log file (aka user_log), if enabled.
+        log_subprocess = subprocess_logger.debug
+        used_level = logging.DEBUG
 
     # Whether the subprocess will be visible in the console.
-    showing_subprocess = True
+    showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level
 
     command_desc = format_command_args(cmd)
     try:
@@ -176,8 +193,10 @@ def call_subprocess(
             raise SubProcessError(exc_msg)
         elif on_returncode == 'warn':
             subprocess_logger.warning(
-                'Command "{}" had error code {} in {}'.format(
-                    command_desc, proc.returncode, cwd)
+                'Command "%s" had error code %s in %s',
+                command_desc,
+                proc.returncode,
+                cwd,
             )
         elif on_returncode == 'ignore':
             pass
@@ -775,9 +794,9 @@ def get_revision(cls, location):
     def run_command(
         cls,
         cmd,  # type: Union[List[str], CommandArgs]
+        show_stdout=True,  # type: bool
         cwd=None,  # type: Optional[str]
         on_returncode='raise',  # type: str
-        extra_environ=None,  # type: Optional[Mapping[str, Any]]
         extra_ok_returncodes=None,  # type: Optional[Iterable[int]]
         log_failed_cmd=True  # type: bool
     ):
@@ -789,9 +808,8 @@ def run_command(
         """
         cmd = make_command(cls.name, *cmd)
         try:
-            return call_subprocess(cmd, cwd,
+            return call_subprocess(cmd, show_stdout, cwd,
                                    on_returncode=on_returncode,
-                                   extra_environ=extra_environ,
                                    extra_ok_returncodes=extra_ok_returncodes,
                                    log_failed_cmd=log_failed_cmd)
         except OSError as e:

From 7e0dcd6353ffa7f28e061e41ee54283ffc527ecd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Mon, 21 Dec 2020 13:03:11 +0100
Subject: [PATCH 08/16] Revert "Create call_subprocess just for vcs commands"

This reverts commit 8adbc216a647b6b349f1b7f1eaa9e71cd3108955.
---
 src/pip/_internal/exceptions.py         |   5 -
 src/pip/_internal/vcs/subversion.py     |  14 ++-
 src/pip/_internal/vcs/versioncontrol.py | 144 +++---------------------
 3 files changed, 25 insertions(+), 138 deletions(-)

diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py
index 3f2659e8792..8d7f7fa39be 100644
--- a/src/pip/_internal/exceptions.py
+++ b/src/pip/_internal/exceptions.py
@@ -91,11 +91,6 @@ class CommandError(PipError):
     """Raised when there is an error in command-line arguments"""
 
 
-class SubProcessError(PipError):
-    """Raised when there is an error raised while executing a
-    command in subprocess"""
-
-
 class PreviousBuildDirError(PipError):
     """Raised when there's a previous conflicting build directory"""
 
diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py
index a57e19751fa..2575872a7b3 100644
--- a/src/pip/_internal/vcs/subversion.py
+++ b/src/pip/_internal/vcs/subversion.py
@@ -25,7 +25,7 @@
 
 
 if MYPY_CHECK_RUNNING:
-    from typing import Optional, Tuple
+    from typing import Optional, Text, Tuple
 
     from pip._internal.utils.misc import HiddenText
     from pip._internal.utils.subprocess import CommandArgs
@@ -218,7 +218,17 @@ def call_vcs_version(self):
         #   svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0)
         #      compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2
         version_prefix = 'svn, version '
-        version = self.run_command(['--version'], show_stdout=True)
+        cmd_output = self.run_command(['--version'], show_stdout=False)
+
+        # Split the output by newline, and find the first line where
+        # version_prefix is present
+        output_lines = cmd_output.split('\n')
+        version = ''  # type: Text
+
+        for line in output_lines:
+            if version_prefix in line:
+                version = line
+                break
 
         if not version.startswith(version_prefix):
             return ()
diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py
index a4995c88f78..1d9cb08feb3 100644
--- a/src/pip/_internal/vcs/versioncontrol.py
+++ b/src/pip/_internal/vcs/versioncontrol.py
@@ -6,15 +6,13 @@
 import logging
 import os
 import shutil
-import subprocess
 import sys
 
 from pip._vendor import pkg_resources
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 
-from pip._internal.exceptions import BadCommand, InstallationError, SubProcessError
-from pip._internal.utils.compat import console_to_str, samefile
-from pip._internal.utils.logging import subprocess_logger
+from pip._internal.exceptions import BadCommand, InstallationError
+from pip._internal.utils.compat import samefile
 from pip._internal.utils.misc import (
     ask_path_exists,
     backup_dir,
@@ -23,21 +21,18 @@
     hide_value,
     rmtree,
 )
-from pip._internal.utils.subprocess import (
-    format_command_args,
-    make_command,
-    make_subprocess_output_error,
-    reveal_command_args,
-)
+from pip._internal.utils.subprocess import call_subprocess, make_command
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.urls import get_url_scheme
 
 if MYPY_CHECK_RUNNING:
     from typing import (
+        Any,
         Dict,
         Iterable,
         Iterator,
         List,
+        Mapping,
         Optional,
         Text,
         Tuple,
@@ -45,6 +40,7 @@
         Union,
     )
 
+    from pip._internal.cli.spinners import SpinnerInterface
     from pip._internal.utils.misc import HiddenText
     from pip._internal.utils.subprocess import CommandArgs
 
@@ -85,127 +81,6 @@ def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None):
     return req
 
 
-def call_subprocess(
-    cmd,  # type: Union[List[str], CommandArgs]
-    show_stdout=False,  # type: bool
-    cwd=None,  # type: Optional[str]
-    on_returncode='raise',  # type: str
-    extra_ok_returncodes=None,  # type: Optional[Iterable[int]]
-    log_failed_cmd=True  # type: Optional[bool]
-):
-    # type: (...) -> Text
-    """
-    Args:
-      show_stdout: if true, use INFO to log the subprocess's stderr and
-        stdout streams.  Otherwise, use DEBUG.  Defaults to False.
-      extra_ok_returncodes: an iterable of integer return codes that are
-        acceptable, in addition to 0. Defaults to None, which means [].
-      log_failed_cmd: if false, failed commands are not logged,
-        only raised.
-    """
-    if extra_ok_returncodes is None:
-        extra_ok_returncodes = []
-    # Most places in pip use show_stdout=False.
-    # What this means is--
-    #
-    # - We log this output of stdout and stderr at DEBUG level
-    # as it is received.
-    # - If DEBUG logging isn't enabled (e.g. if --verbose logging wasn't
-    #   requested), then we show a spinner so the user can still see the
-    #   subprocess is in progress.
-    # - If the subprocess exits with an error, we log the output to stderr
-    #   at ERROR level if it hasn't already been displayed to the console
-    #   (e.g. if --verbose logging wasn't enabled).  This way we don't log
-    #   the output to the console twice.
-    #
-    # If show_stdout=True, then the above is still done, but with DEBUG
-    # replaced by INFO.
-    if show_stdout:
-        # Then log the subprocess output at INFO level.
-        log_subprocess = subprocess_logger.info
-        used_level = logging.INFO
-    else:
-        # Then log the subprocess output using DEBUG.  This also ensures
-        # it will be logged to the log file (aka user_log), if enabled.
-        log_subprocess = subprocess_logger.debug
-        used_level = logging.DEBUG
-
-    # Whether the subprocess will be visible in the console.
-    showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level
-
-    command_desc = format_command_args(cmd)
-    try:
-        proc = subprocess.Popen(
-            # Convert HiddenText objects to the underlying str.
-            reveal_command_args(cmd),
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            cwd=cwd
-        )
-        if proc.stdin:
-            proc.stdin.close()
-    except Exception as exc:
-        if log_failed_cmd:
-            subprocess_logger.critical(
-                "Error %s while executing command %s", exc, command_desc,
-            )
-        raise
-    all_output = []
-    while True:
-        # The "line" value is a unicode string in Python 2.
-        line = None
-        if proc.stdout:
-            line = console_to_str(proc.stdout.readline())
-        if not line:
-            break
-        line = line.rstrip()
-        all_output.append(line + '\n')
-
-        # Show the line immediately.
-        log_subprocess(line)
-    try:
-        proc.wait()
-    finally:
-        if proc.stdout:
-            proc.stdout.close()
-        if proc.stderr:
-            proc.stderr.close()
-
-    proc_had_error = (
-        proc.returncode and proc.returncode not in extra_ok_returncodes
-    )
-    if proc_had_error:
-        if on_returncode == 'raise':
-            if not showing_subprocess and log_failed_cmd:
-                # Then the subprocess streams haven't been logged to the
-                # console yet.
-                msg = make_subprocess_output_error(
-                    cmd_args=cmd,
-                    cwd=cwd,
-                    lines=all_output,
-                    exit_status=proc.returncode,
-                )
-                subprocess_logger.error(msg)
-            exc_msg = (
-                'Command errored out with exit status {}: {} '
-                'Check the logs for full command output.'
-            ).format(proc.returncode, command_desc)
-            raise SubProcessError(exc_msg)
-        elif on_returncode == 'warn':
-            subprocess_logger.warning(
-                'Command "%s" had error code %s in %s',
-                command_desc,
-                proc.returncode,
-                cwd,
-            )
-        elif on_returncode == 'ignore':
-            pass
-        else:
-            raise ValueError('Invalid value: on_returncode={!r}'.format(
-                             on_returncode))
-    return ''.join(all_output)
-
-
 def find_path_to_setup_from_repo_root(location, repo_root):
     # type: (str, str) -> Optional[str]
     """
@@ -798,6 +673,9 @@ def run_command(
         cwd=None,  # type: Optional[str]
         on_returncode='raise',  # type: str
         extra_ok_returncodes=None,  # type: Optional[Iterable[int]]
+        command_desc=None,  # type: Optional[str]
+        extra_environ=None,  # type: Optional[Mapping[str, Any]]
+        spinner=None,  # type: Optional[SpinnerInterface]
         log_failed_cmd=True  # type: bool
     ):
         # type: (...) -> Text
@@ -811,6 +689,10 @@ def run_command(
             return call_subprocess(cmd, show_stdout, cwd,
                                    on_returncode=on_returncode,
                                    extra_ok_returncodes=extra_ok_returncodes,
+                                   command_desc=command_desc,
+                                   extra_environ=extra_environ,
+                                   unset_environ=cls.unset_environ,
+                                   spinner=spinner,
                                    log_failed_cmd=log_failed_cmd)
         except OSError as e:
             # errno.ENOENT = no such file or directory

From 624398ad7248469726a026ca12370cd7d4d1c3b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Mon, 21 Dec 2020 13:05:16 +0100
Subject: [PATCH 09/16] Revert "Improve check for svn version string"

This reverts commit 1471897b84b43c467c753b5edebe636f835afc6a.
---
 src/pip/_internal/vcs/subversion.py | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py
index 2575872a7b3..b84108efc52 100644
--- a/src/pip/_internal/vcs/subversion.py
+++ b/src/pip/_internal/vcs/subversion.py
@@ -25,7 +25,7 @@
 
 
 if MYPY_CHECK_RUNNING:
-    from typing import Optional, Text, Tuple
+    from typing import Optional, Tuple
 
     from pip._internal.utils.misc import HiddenText
     from pip._internal.utils.subprocess import CommandArgs
@@ -218,18 +218,7 @@ def call_vcs_version(self):
         #   svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0)
         #      compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2
         version_prefix = 'svn, version '
-        cmd_output = self.run_command(['--version'], show_stdout=False)
-
-        # Split the output by newline, and find the first line where
-        # version_prefix is present
-        output_lines = cmd_output.split('\n')
-        version = ''  # type: Text
-
-        for line in output_lines:
-            if version_prefix in line:
-                version = line
-                break
-
+        version = self.run_command(['--version'], show_stdout=False)
         if not version.startswith(version_prefix):
             return ()
 

From 49d34ceb63fa9f201acbfd19eda6e5998f5500b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Mon, 21 Dec 2020 15:07:48 +0100
Subject: [PATCH 10/16] Revert "Bubble up SubProcessError to basecommand._main"

This reverts commit e9f738a3daec91b131ae985e16809d47b1cfdaff.
---
 src/pip/_internal/cli/base_command.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py
index 7f05efb85db..41e7dcf101b 100644
--- a/src/pip/_internal/cli/base_command.py
+++ b/src/pip/_internal/cli/base_command.py
@@ -27,7 +27,6 @@
     InstallationError,
     NetworkConnectionError,
     PreviousBuildDirError,
-    SubProcessError,
     UninstallationError,
 )
 from pip._internal.utils.deprecation import deprecated
@@ -230,7 +229,7 @@ def _main(self, args):
 
             return PREVIOUS_BUILD_DIR_ERROR
         except (InstallationError, UninstallationError, BadCommand,
-                SubProcessError, NetworkConnectionError) as exc:
+                NetworkConnectionError) as exc:
             logger.critical(str(exc))
             logger.debug('Exception information:', exc_info=True)
 

From 27d720d11e9a641a07ea0e5a35b6b0f933730a37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Mon, 21 Dec 2020 13:33:46 +0100
Subject: [PATCH 11/16] Additional revert of 7969

Revert additional changes that were made
after 7969 and depended on it.
---
 src/pip/_internal/vcs/git.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index a6d1396fc77..de9c7f51981 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -11,7 +11,7 @@
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 from pip._vendor.six.moves.urllib import request as urllib_request
 
-from pip._internal.exceptions import BadCommand, InstallationError, SubProcessError
+from pip._internal.exceptions import BadCommand, InstallationError
 from pip._internal.utils.misc import display_path, hide_url
 from pip._internal.utils.subprocess import make_command
 from pip._internal.utils.temp_dir import TempDirectory
@@ -332,9 +332,11 @@ def has_commit(cls, location, rev):
         """
         try:
             cls.run_command(
-                ['rev-parse', '-q', '--verify', "sha^" + rev], cwd=location
+                ['rev-parse', '-q', '--verify', "sha^" + rev],
+                cwd=location,
+                log_failed_cmd=False,
             )
-        except SubProcessError:
+        except InstallationError:
             return False
         else:
             return True

From efc67f0ab54f4aa9885000e665f9c370a3a9cd4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Mon, 21 Dec 2020 14:39:50 +0100
Subject: [PATCH 12/16] add stdout_only to call_subprocess

---
 src/pip/_internal/utils/subprocess.py | 69 ++++++++++++++++++---------
 1 file changed, 47 insertions(+), 22 deletions(-)

diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py
index 325897c8739..4c73c68377d 100644
--- a/src/pip/_internal/utils/subprocess.py
+++ b/src/pip/_internal/utils/subprocess.py
@@ -118,7 +118,8 @@ def call_subprocess(
     extra_environ=None,  # type: Optional[Mapping[str, Any]]
     unset_environ=None,  # type: Optional[Iterable[str]]
     spinner=None,  # type: Optional[SpinnerInterface]
-    log_failed_cmd=True  # type: Optional[bool]
+    log_failed_cmd=True,  # type: Optional[bool]
+    stdout_only=False,  # type: Optional[bool]
 ):
     # type: (...) -> Text
     """
@@ -130,6 +131,9 @@ def call_subprocess(
       unset_environ: an iterable of environment variable names to unset
         prior to calling subprocess.Popen().
       log_failed_cmd: if false, failed commands are not logged, only raised.
+      stdout_only: if true, return only stdout, else return both. When true,
+        logging of both stdout and stderr occurs when the subprocess has
+        terminated, else logging occurs as subprocess output is produced.
     """
     if extra_ok_returncodes is None:
         extra_ok_returncodes = []
@@ -180,8 +184,11 @@ def call_subprocess(
         proc = subprocess.Popen(
             # Convert HiddenText objects to the underlying str.
             reveal_command_args(cmd),
-            stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE, cwd=cwd, env=env,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT if not stdout_only else subprocess.PIPE,
+            cwd=cwd,
+            env=env,
         )
         assert proc.stdin
         assert proc.stdout
@@ -193,25 +200,43 @@ def call_subprocess(
             )
         raise
     all_output = []
-    while True:
-        # The "line" value is a unicode string in Python 2.
-        line = console_to_str(proc.stdout.readline())
-        if not line:
-            break
-        line = line.rstrip()
-        all_output.append(line + '\n')
+    if not stdout_only:
+        # In this mode, stdout and stderr are in the same pip.
+        while True:
+            # The "line" value is a unicode string in Python 2.
+            line = console_to_str(proc.stdout.readline())
+            if not line:
+                break
+            line = line.rstrip()
+            all_output.append(line + '\n')
+
+            # Show the line immediately.
+            log_subprocess(line)
+            # Update the spinner.
+            if use_spinner:
+                assert spinner
+                spinner.spin()
+        try:
+            proc.wait()
+        finally:
+            if proc.stdout:
+                proc.stdout.close()
+        output = ''.join(all_output)
+    else:
+        # In this mode, stdout and stderr are in different pipes.
+        # We must use the communicate which is the only safe way to read both.
+        out_bytes, err_bytes = proc.communicate()
+        # log line by line to preserve pip log indenting
+        out = console_to_str(out_bytes)
+        for out_line in out.splitlines():
+            log_subprocess(out_line)
+        all_output.append(out)
+        err = console_to_str(err_bytes)
+        for err_line in err.splitlines():
+            log_subprocess(err_line)
+        all_output.append(err)
+        output = out
 
-        # Show the line immediately.
-        log_subprocess(line)
-        # Update the spinner.
-        if use_spinner:
-            assert spinner
-            spinner.spin()
-    try:
-        proc.wait()
-    finally:
-        if proc.stdout:
-            proc.stdout.close()
     proc_had_error = (
         proc.returncode and proc.returncode not in extra_ok_returncodes
     )
@@ -246,7 +271,7 @@ def call_subprocess(
         else:
             raise ValueError('Invalid value: on_returncode={!r}'.format(
                              on_returncode))
-    return ''.join(all_output)
+    return output
 
 
 def runner_with_spinner_message(message):

From daca25df0c4fb1e84ef34932d6b815ea203b397a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Mon, 21 Dec 2020 14:46:21 +0100
Subject: [PATCH 13/16] vcs: capture subprocess stdout only

---
 news/8876.bugfix.rst                    |  3 +++
 src/pip/_internal/utils/subprocess.py   | 10 +++----
 src/pip/_internal/vcs/bazaar.py         |  6 +++--
 src/pip/_internal/vcs/git.py            | 35 ++++++++++++++++++++-----
 src/pip/_internal/vcs/mercurial.py      | 19 +++++++++++---
 src/pip/_internal/vcs/subversion.py     |  5 +++-
 src/pip/_internal/vcs/versioncontrol.py |  6 +++--
 7 files changed, 63 insertions(+), 21 deletions(-)
 create mode 100644 news/8876.bugfix.rst

diff --git a/news/8876.bugfix.rst b/news/8876.bugfix.rst
new file mode 100644
index 00000000000..98250dc9745
--- /dev/null
+++ b/news/8876.bugfix.rst
@@ -0,0 +1,3 @@
+Fixed hanging VCS subprocess calls when the VCS outputs a large amount of data
+on stderr. Restored logging of VCS errors that was inadvertently removed in pip
+20.2.
diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py
index 4c73c68377d..3cd8b01f73e 100644
--- a/src/pip/_internal/utils/subprocess.py
+++ b/src/pip/_internal/utils/subprocess.py
@@ -190,9 +190,6 @@ def call_subprocess(
             cwd=cwd,
             env=env,
         )
-        assert proc.stdin
-        assert proc.stdout
-        proc.stdin.close()
     except Exception as exc:
         if log_failed_cmd:
             subprocess_logger.critical(
@@ -201,7 +198,10 @@ def call_subprocess(
         raise
     all_output = []
     if not stdout_only:
-        # In this mode, stdout and stderr are in the same pip.
+        assert proc.stdout
+        assert proc.stdin
+        proc.stdin.close()
+        # In this mode, stdout and stderr are in the same pipe.
         while True:
             # The "line" value is a unicode string in Python 2.
             line = console_to_str(proc.stdout.readline())
@@ -224,7 +224,7 @@ def call_subprocess(
         output = ''.join(all_output)
     else:
         # In this mode, stdout and stderr are in different pipes.
-        # We must use the communicate which is the only safe way to read both.
+        # We must use communicate() which is the only safe way to read both.
         out_bytes, err_bytes = proc.communicate()
         # log line by line to preserve pip log indenting
         out = console_to_str(out_bytes)
diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py
index efe524492af..4a63d6faa5c 100644
--- a/src/pip/_internal/vcs/bazaar.py
+++ b/src/pip/_internal/vcs/bazaar.py
@@ -93,7 +93,9 @@ def get_url_rev_and_auth(cls, url):
 
     @classmethod
     def get_remote_url(cls, location):
-        urls = cls.run_command(['info'], show_stdout=False, cwd=location)
+        urls = cls.run_command(
+            ['info'], show_stdout=False, stdout_only=True, cwd=location
+        )
         for line in urls.splitlines():
             line = line.strip()
             for x in ('checkout of branch: ',
@@ -108,7 +110,7 @@ def get_remote_url(cls, location):
     @classmethod
     def get_revision(cls, location):
         revision = cls.run_command(
-            ['revno'], show_stdout=False, cwd=location,
+            ['revno'], show_stdout=False, stdout_only=True, cwd=location,
         )
         return revision.splitlines()[-1]
 
diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index de9c7f51981..565961a0631 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -79,7 +79,9 @@ def is_immutable_rev_checkout(self, url, dest):
 
     def get_git_version(self):
         VERSION_PFX = 'git version '
-        version = self.run_command(['version'], show_stdout=False)
+        version = self.run_command(
+            ['version'], show_stdout=False, stdout_only=True
+        )
         if version.startswith(VERSION_PFX):
             version = version[len(VERSION_PFX):].split()[0]
         else:
@@ -102,7 +104,11 @@ def get_current_branch(cls, location):
         # and to suppress the message to stderr.
         args = ['symbolic-ref', '-q', 'HEAD']
         output = cls.run_command(
-            args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location,
+            args,
+            extra_ok_returncodes=(1, ),
+            show_stdout=False,
+            stdout_only=True,
+            cwd=location,
         )
         ref = output.strip()
 
@@ -135,8 +141,13 @@ def get_revision_sha(cls, dest, rev):
           rev: the revision name.
         """
         # Pass rev to pre-filter the list.
-        output = cls.run_command(['show-ref', rev], cwd=dest,
-                                 show_stdout=False, on_returncode='ignore')
+        output = cls.run_command(
+            ['show-ref', rev],
+            cwd=dest,
+            show_stdout=False,
+            stdout_only=True,
+            on_returncode='ignore',
+        )
         refs = {}
         for line in output.strip().splitlines():
             try:
@@ -310,7 +321,10 @@ def get_remote_url(cls, location):
         # exits with return code 1 if there are no matching lines.
         stdout = cls.run_command(
             ['config', '--get-regexp', r'remote\..*\.url'],
-            extra_ok_returncodes=(1, ), show_stdout=False, cwd=location,
+            extra_ok_returncodes=(1, ),
+            show_stdout=False,
+            stdout_only=True,
+            cwd=location,
         )
         remotes = stdout.splitlines()
         try:
@@ -346,7 +360,10 @@ def get_revision(cls, location, rev=None):
         if rev is None:
             rev = 'HEAD'
         current_rev = cls.run_command(
-            ['rev-parse', rev], show_stdout=False, cwd=location,
+            ['rev-parse', rev],
+            show_stdout=False,
+            stdout_only=True,
+            cwd=location,
         )
         return current_rev.strip()
 
@@ -359,7 +376,10 @@ def get_subdirectory(cls, location):
         # find the repo root
         git_dir = cls.run_command(
             ['rev-parse', '--git-dir'],
-            show_stdout=False, cwd=location).strip()
+            show_stdout=False,
+            stdout_only=True,
+            cwd=location,
+        ).strip()
         if not os.path.isabs(git_dir):
             git_dir = os.path.join(location, git_dir)
         repo_root = os.path.abspath(os.path.join(git_dir, '..'))
@@ -418,6 +438,7 @@ def get_repository_root(cls, location):
                 ['rev-parse', '--show-toplevel'],
                 cwd=location,
                 show_stdout=False,
+                stdout_only=True,
                 on_returncode='raise',
                 log_failed_cmd=False,
             )
diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py
index 75e903cc8a6..d2d145f623f 100644
--- a/src/pip/_internal/vcs/mercurial.py
+++ b/src/pip/_internal/vcs/mercurial.py
@@ -92,7 +92,10 @@ def update(self, dest, url, rev_options):
     def get_remote_url(cls, location):
         url = cls.run_command(
             ['showconfig', 'paths.default'],
-            show_stdout=False, cwd=location).strip()
+            show_stdout=False,
+            stdout_only=True,
+            cwd=location,
+        ).strip()
         if cls._is_local_repository(url):
             url = path_to_url(url)
         return url.strip()
@@ -104,7 +107,10 @@ def get_revision(cls, location):
         """
         current_revision = cls.run_command(
             ['parents', '--template={rev}'],
-            show_stdout=False, cwd=location).strip()
+            show_stdout=False,
+            stdout_only=True,
+            cwd=location,
+        ).strip()
         return current_revision
 
     @classmethod
@@ -115,7 +121,10 @@ def get_requirement_revision(cls, location):
         """
         current_rev_hash = cls.run_command(
             ['parents', '--template={node}'],
-            show_stdout=False, cwd=location).strip()
+            show_stdout=False,
+            stdout_only=True,
+            cwd=location,
+        ).strip()
         return current_rev_hash
 
     @classmethod
@@ -131,7 +140,8 @@ def get_subdirectory(cls, location):
         """
         # find the repo root
         repo_root = cls.run_command(
-            ['root'], show_stdout=False, cwd=location).strip()
+            ['root'], show_stdout=False, stdout_only=True, cwd=location
+        ).strip()
         if not os.path.isabs(repo_root):
             repo_root = os.path.abspath(os.path.join(location, repo_root))
         return find_path_to_setup_from_repo_root(location, repo_root)
@@ -146,6 +156,7 @@ def get_repository_root(cls, location):
                 ['root'],
                 cwd=location,
                 show_stdout=False,
+                stdout_only=True,
                 on_returncode='raise',
                 log_failed_cmd=False,
             )
diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py
index b84108efc52..701f41db4b2 100644
--- a/src/pip/_internal/vcs/subversion.py
+++ b/src/pip/_internal/vcs/subversion.py
@@ -167,6 +167,7 @@ def _get_svn_url_rev(cls, location):
                 xml = cls.run_command(
                     ['info', '--xml', location],
                     show_stdout=False,
+                    stdout_only=True,
                 )
                 url = _svn_info_xml_url_re.search(xml).group(1)
                 revs = [
@@ -218,7 +219,9 @@ def call_vcs_version(self):
         #   svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0)
         #      compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2
         version_prefix = 'svn, version '
-        version = self.run_command(['--version'], show_stdout=False)
+        version = self.run_command(
+            ['--version'], show_stdout=False, stdout_only=True
+        )
         if not version.startswith(version_prefix):
             return ()
 
diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py
index 1d9cb08feb3..0e807a2fb06 100644
--- a/src/pip/_internal/vcs/versioncontrol.py
+++ b/src/pip/_internal/vcs/versioncontrol.py
@@ -676,7 +676,8 @@ def run_command(
         command_desc=None,  # type: Optional[str]
         extra_environ=None,  # type: Optional[Mapping[str, Any]]
         spinner=None,  # type: Optional[SpinnerInterface]
-        log_failed_cmd=True  # type: bool
+        log_failed_cmd=True,  # type: bool
+        stdout_only=False,  # type: bool
     ):
         # type: (...) -> Text
         """
@@ -693,7 +694,8 @@ def run_command(
                                    extra_environ=extra_environ,
                                    unset_environ=cls.unset_environ,
                                    spinner=spinner,
-                                   log_failed_cmd=log_failed_cmd)
+                                   log_failed_cmd=log_failed_cmd,
+                                   stdout_only=stdout_only)
         except OSError as e:
             # errno.ENOENT = no such file or directory
             # In other words, the VCS executable isn't available

From ac4c53868a3680fc36e686a6a2c845700b2ebb12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Tue, 22 Dec 2020 00:16:42 +0100
Subject: [PATCH 14/16] Add test for call_subprocess stdout_only

---
 tests/unit/test_utils_subprocess.py | 30 +++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/tests/unit/test_utils_subprocess.py b/tests/unit/test_utils_subprocess.py
index 1a031065114..b6c2eb63c7e 100644
--- a/tests/unit/test_utils_subprocess.py
+++ b/tests/unit/test_utils_subprocess.py
@@ -14,6 +14,7 @@
     format_command_args,
     make_command,
     make_subprocess_output_error,
+    subprocess_logger,
 )
 
 
@@ -154,6 +155,35 @@ def test_make_subprocess_output_error__non_ascii_line():
     assert actual == expected, u'actual: {}'.format(actual)
 
 
+@pytest.mark.parametrize(
+    ('stdout_only', 'expected'),
+    [
+        (True, ("out\n", "out\r\n")),
+        (False, ("out\nerr\n", "out\r\nerr\r\n", "err\nout\n", "err\r\nout\r\n")),
+    ],
+)
+def test_call_subprocess_stdout_only(capfd, monkeypatch, stdout_only, expected):
+    log = []
+    monkeypatch.setattr(subprocess_logger, "debug", lambda *args: log.append(args[0]))
+    out = call_subprocess(
+        [
+            sys.executable,
+            "-c",
+            "import sys; "
+            "sys.stdout.write('out\\n'); "
+            "sys.stderr.write('err\\n')"
+        ],
+        stdout_only=stdout_only,
+    )
+    assert out in expected
+    captured = capfd.readouterr()
+    assert captured.err == ""
+    assert (
+        log == ["Running command %s", "out", "err"]
+        or log == ["Running command %s", "err", "out"]
+    )
+
+
 class FakeSpinner(SpinnerInterface):
 
     def __init__(self):

From f0f67af32c8f58031d6c239d46899fb824b94262 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@users.noreply.github.com>
Date: Sat, 23 Jan 2021 11:51:53 +0000
Subject: [PATCH 15/16] Bump for release

---
 NEWS.rst                   | 33 +++++++++++++++++++++++++++++++++
 news/8876.bugfix.rst       |  3 ---
 news/9180.bugfix.rst       |  1 -
 news/9203.bugfix.rst       |  4 ----
 news/9206.feature.rst      |  3 ---
 news/9246.bugfix.rst       |  4 ----
 news/9315.feature.rst      |  1 -
 news/resolvelib.vendor.rst |  1 -
 src/pip/__init__.py        |  2 +-
 9 files changed, 34 insertions(+), 18 deletions(-)
 delete mode 100644 news/8876.bugfix.rst
 delete mode 100644 news/9180.bugfix.rst
 delete mode 100644 news/9203.bugfix.rst
 delete mode 100644 news/9206.feature.rst
 delete mode 100644 news/9246.bugfix.rst
 delete mode 100644 news/9315.feature.rst
 delete mode 100644 news/resolvelib.vendor.rst

diff --git a/NEWS.rst b/NEWS.rst
index a082cddf314..18cc7b70427 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -9,6 +9,39 @@
 
 .. towncrier release notes start
 
+20.3.4 (2021-01-23)
+===================
+
+Features
+--------
+
+- ``pip wheel`` now verifies the built wheel contains valid metadata, and can be
+  installed by a subsequent ``pip install``. This can be disabled with
+  ``--no-verify``. (`#9206 <https://github.com/pypa/pip/issues/9206>`_)
+- Improve presentation of XMLRPC errors in pip search. (`#9315 <https://github.com/pypa/pip/issues/9315>`_)
+
+Bug Fixes
+---------
+
+- Fixed hanging VCS subprocess calls when the VCS outputs a large amount of data
+  on stderr. Restored logging of VCS errors that was inadvertently removed in pip
+  20.2. (`#8876 <https://github.com/pypa/pip/issues/8876>`_)
+- Fix error when an existing incompatibility is unable to be applied to a backtracked state. (`#9180 <https://github.com/pypa/pip/issues/9180>`_)
+- New resolver: Discard a faulty distribution, instead of quitting outright.
+  This implementation is taken from 20.2.2, with a fix that always makes the
+  resolver iterate through candidates from indexes lazily, to avoid downloading
+  candidates we do not need. (`#9203 <https://github.com/pypa/pip/issues/9203>`_)
+- New resolver: Discard a source distribution if it fails to generate metadata,
+  instead of quitting outright. This implementation is taken from 20.2.2, with a
+  fix that always makes the resolver iterate through candidates from indexes
+  lazily, to avoid downloading candidates we do not need. (`#9246 <https://github.com/pypa/pip/issues/9246>`_)
+
+Vendored Libraries
+------------------
+
+- Upgrade resolvelib to 0.5.4.
+
+
 20.3.3 (2020-12-15)
 ===================
 
diff --git a/news/8876.bugfix.rst b/news/8876.bugfix.rst
deleted file mode 100644
index 98250dc9745..00000000000
--- a/news/8876.bugfix.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fixed hanging VCS subprocess calls when the VCS outputs a large amount of data
-on stderr. Restored logging of VCS errors that was inadvertently removed in pip
-20.2.
diff --git a/news/9180.bugfix.rst b/news/9180.bugfix.rst
deleted file mode 100644
index e597c1ad90a..00000000000
--- a/news/9180.bugfix.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix error when an existing incompatibility is unable to be applied to a backtracked state.
diff --git a/news/9203.bugfix.rst b/news/9203.bugfix.rst
deleted file mode 100644
index 38320218fbb..00000000000
--- a/news/9203.bugfix.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-New resolver: Discard a faulty distribution, instead of quitting outright.
-This implementation is taken from 20.2.2, with a fix that always makes the
-resolver iterate through candidates from indexes lazily, to avoid downloading
-candidates we do not need.
diff --git a/news/9206.feature.rst b/news/9206.feature.rst
deleted file mode 100644
index 90cd2cf99fb..00000000000
--- a/news/9206.feature.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-``pip wheel`` now verifies the built wheel contains valid metadata, and can be
-installed by a subsequent ``pip install``. This can be disabled with
-``--no-verify``.
diff --git a/news/9246.bugfix.rst b/news/9246.bugfix.rst
deleted file mode 100644
index 93f7f18f9f5..00000000000
--- a/news/9246.bugfix.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-New resolver: Discard a source distribution if it fails to generate metadata,
-instead of quitting outright. This implementation is taken from 20.2.2, with a
-fix that always makes the resolver iterate through candidates from indexes
-lazily, to avoid downloading candidates we do not need.
diff --git a/news/9315.feature.rst b/news/9315.feature.rst
deleted file mode 100644
index 64d1f25338b..00000000000
--- a/news/9315.feature.rst
+++ /dev/null
@@ -1 +0,0 @@
-Improve presentation of XMLRPC errors in pip search.
diff --git a/news/resolvelib.vendor.rst b/news/resolvelib.vendor.rst
deleted file mode 100644
index 680da3be1e7..00000000000
--- a/news/resolvelib.vendor.rst
+++ /dev/null
@@ -1 +0,0 @@
-Upgrade resolvelib to 0.5.4.
diff --git a/src/pip/__init__.py b/src/pip/__init__.py
index 41b291e61a0..f9bd0632fe2 100644
--- a/src/pip/__init__.py
+++ b/src/pip/__init__.py
@@ -4,7 +4,7 @@
     from typing import List, Optional
 
 
-__version__ = "20.3.3"
+__version__ = "20.3.4"
 
 
 def main(args=None):

From fdba3faa982625c757b727251d27c6a1c92336b2 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@users.noreply.github.com>
Date: Sat, 23 Jan 2021 11:51:53 +0000
Subject: [PATCH 16/16] Bump for development

---
 src/pip/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pip/__init__.py b/src/pip/__init__.py
index f9bd0632fe2..ae0fe9a9f24 100644
--- a/src/pip/__init__.py
+++ b/src/pip/__init__.py
@@ -4,7 +4,7 @@
     from typing import List, Optional
 
 
-__version__ = "20.3.4"
+__version__ = "21.0.dev0"
 
 
 def main(args=None):