Merge pull request #8026 from sbidoul/requested-sbi

news/7811.feature

@@ -0,0 +1,2 @@
+Generate PEP 376 REQUESTED metadata for user supplied requirements installed
+by pip.

src/pip/_internal/cli/req_command.py

@@ -309,25 +309,25 @@ def get_requirements(
                 req_to_add = install_req_from_parsed_requirement(
                     parsed_req,
                     isolated=options.isolated_mode,
+                    user_supplied=False,
                 )
-                req_to_add.is_direct = True
                 requirements.append(req_to_add)
 
         for req in args:
             req_to_add = install_req_from_line(
                 req, None, isolated=options.isolated_mode,
                 use_pep517=options.use_pep517,
+                user_supplied=True,
             )
-            req_to_add.is_direct = True
             requirements.append(req_to_add)
 
         for req in options.editables:
             req_to_add = install_req_from_editable(
                 req,
+                user_supplied=True,
                 isolated=options.isolated_mode,
                 use_pep517=options.use_pep517,
             )
-            req_to_add.is_direct = True
             requirements.append(req_to_add)
 
         # NOTE: options.require_hashes may be set if --require-hashes is True
@@ -338,9 +338,9 @@ def get_requirements(
                 req_to_add = install_req_from_parsed_requirement(
                     parsed_req,
                     isolated=options.isolated_mode,
-                    use_pep517=options.use_pep517
+                    use_pep517=options.use_pep517,
+                    user_supplied=True,
                 )
-                req_to_add.is_direct = True
                 requirements.append(req_to_add)
 
         # If any requirement has hash options, enable hash checking.

src/pip/_internal/operations/install/wheel.py

@@ -339,6 +339,7 @@ def install_unpacked_wheel(
     pycompile=True,  # type: bool
     warn_script_location=True,  # type: bool
     direct_url=None,  # type: Optional[DirectUrl]
+    requested=False,  # type: bool
 ):
     # type: (...) -> None
     """Install a wheel.
@@ -645,6 +646,13 @@ def _generate_file(path, **kwargs):
             direct_url_file.write(direct_url.to_json().encode("utf-8"))
         generated.append(direct_url_path)
 
+    # Record the REQUESTED file
+    if requested:
+        requested_path = os.path.join(dest_info_dir, 'REQUESTED')
+        with open(requested_path, "w"):
+            pass
+        generated.append(requested_path)
+
     # Record details of all files installed
     record_path = os.path.join(dest_info_dir, 'RECORD')
     with open(record_path, **csv_io_kwargs('r')) as record_file:
@@ -671,6 +679,7 @@ def install_wheel(
     warn_script_location=True,  # type: bool
     _temp_dir_for_testing=None,  # type: Optional[str]
     direct_url=None,  # type: Optional[DirectUrl]
+    requested=False,  # type: bool
 ):
     # type: (...) -> None
     with TempDirectory(
@@ -686,4 +695,5 @@ def install_wheel(
             pycompile=pycompile,
             warn_script_location=warn_script_location,
             direct_url=direct_url,
+            requested=requested,
         )

src/pip/_internal/req/constructors.py

@@ -219,7 +219,8 @@ def install_req_from_editable(
     use_pep517=None,  # type: Optional[bool]
     isolated=False,  # type: bool
     options=None,  # type: Optional[Dict[str, Any]]
-    constraint=False  # type: bool
+    constraint=False,  # type: bool
+    user_supplied=False,  # type: bool
 ):
     # type: (...) -> InstallRequirement
 
@@ -228,6 +229,7 @@ def install_req_from_editable(
     return InstallRequirement(
         parts.requirement,
         comes_from=comes_from,
+        user_supplied=user_supplied,
         editable=True,
         link=parts.link,
         constraint=constraint,
@@ -382,6 +384,7 @@ def install_req_from_line(
     options=None,  # type: Optional[Dict[str, Any]]
     constraint=False,  # type: bool
     line_source=None,  # type: Optional[str]
+    user_supplied=False,  # type: bool
 ):
     # type: (...) -> InstallRequirement
     """Creates an InstallRequirement from a name, which might be a
@@ -400,14 +403,16 @@ def install_req_from_line(
         hash_options=options.get("hashes", {}) if options else {},
         constraint=constraint,
         extras=parts.extras,
+        user_supplied=user_supplied,
     )
 
 
 def install_req_from_req_string(
     req_string,  # type: str
     comes_from=None,  # type: Optional[InstallRequirement]
     isolated=False,  # type: bool
-    use_pep517=None  # type: Optional[bool]
+    use_pep517=None,  # type: Optional[bool]
+    user_supplied=False,  # type: bool
 ):
     # type: (...) -> InstallRequirement
     try:
@@ -429,14 +434,19 @@ def install_req_from_req_string(
         )
 
     return InstallRequirement(
-        req, comes_from, isolated=isolated, use_pep517=use_pep517
+        req,
+        comes_from,
+        isolated=isolated,
+        use_pep517=use_pep517,
+        user_supplied=user_supplied,
     )
 
 
 def install_req_from_parsed_requirement(
     parsed_req,  # type: ParsedRequirement
     isolated=False,  # type: bool
-    use_pep517=None  # type: Optional[bool]
+    use_pep517=None,  # type: Optional[bool]
+    user_supplied=False,  # type: bool
 ):
     # type: (...) -> InstallRequirement
     if parsed_req.is_editable:
@@ -446,6 +456,7 @@ def install_req_from_parsed_requirement(
             use_pep517=use_pep517,
             constraint=parsed_req.constraint,
             isolated=isolated,
+            user_supplied=user_supplied,
         )
 
     else:
@@ -457,5 +468,6 @@ def install_req_from_parsed_requirement(
             options=parsed_req.options,
             constraint=parsed_req.constraint,
             line_source=parsed_req.line_source,
+            user_supplied=user_supplied,
         )
     return req

src/pip/_internal/req/req_install.py

@@ -111,7 +111,8 @@ def __init__(
         global_options=None,  # type: Optional[List[str]]
         hash_options=None,  # type: Optional[Dict[str, List[str]]]
         constraint=False,  # type: bool
-        extras=()  # type: Iterable[str]
+        extras=(),  # type: Iterable[str]
+        user_supplied=False,  # type: bool
     ):
         # type: (...) -> None
         assert req is None or isinstance(req, Requirement), req
@@ -172,7 +173,10 @@ def __init__(
         self.hash_options = hash_options if hash_options else {}
         # Set to True after successful preparation of this requirement
         self.prepared = False
-        self.is_direct = False
+        # User supplied requirement are explicitly requested for installation
+        # by the user via CLI arguments or requirements files, as opposed to,
+        # e.g. dependencies, extras or constraints.
+        self.user_supplied = user_supplied
 
         # Set by the legacy resolver when the requirement has been downloaded
         # TODO: This introduces a strong coupling between the resolver and the
@@ -820,6 +824,7 @@ def install(
                 pycompile=pycompile,
                 warn_script_location=warn_script_location,
                 direct_url=direct_url,
+                requested=self.user_supplied,
             )
             self.install_succeeded = True
             return

src/pip/_internal/req/req_set.py

@@ -107,9 +107,8 @@ def add_requirement(
                 )
 
         # This next bit is really a sanity check.
-        assert install_req.is_direct == (parent_req_name is None), (
-            "a direct req shouldn't have a parent and also, "
-            "a non direct req should have a parent"
+        assert not install_req.user_supplied or parent_req_name is None, (
+            "a user supplied req shouldn't have a parent"
         )
 
         # Unnamed requirements are scanned again and the requirement won't be
@@ -165,6 +164,10 @@ def add_requirement(
         # If we're now installing a constraint, mark the existing
         # object for real installation.
         existing_req.constraint = False
+        # If we're now installing a user supplied requirement,
+        # mark the existing object as such.
+        if install_req.user_supplied:
+            existing_req.user_supplied = True
         existing_req.extras = tuple(sorted(
             set(existing_req.extras) | set(install_req.extras)
         ))

src/pip/_internal/resolution/legacy/resolver.py

@@ -195,7 +195,7 @@ def _is_upgrade_allowed(self, req):
             return True
         else:
             assert self.upgrade_strategy == "only-if-needed"
-            return req.is_direct
+            return req.user_supplied or req.constraint
 
     def _set_req_to_reinstall(self, req):
         # type: (InstallRequirement) -> None
@@ -419,7 +419,7 @@ def add_req(subreq, extras_requested):
                 # 'unnamed' requirements will get added here
                 # 'unnamed' requirements can only come from being directly
                 # provided by the user.
-                assert req_to_install.is_direct
+                assert req_to_install.user_supplied
                 requirement_set.add_requirement(
                     req_to_install, parent_req_name=None,
                 )

src/pip/_internal/resolution/resolvelib/candidates.py

@@ -48,6 +48,7 @@ def make_install_req_from_link(link, template):
         line = link.url
     ireq = install_req_from_line(
         line,
+        user_supplied=template.user_supplied,
         comes_from=template.comes_from,
         use_pep517=template.use_pep517,
         isolated=template.isolated,
@@ -68,6 +69,7 @@ def make_install_req_from_editable(link, template):
     assert template.editable, "template not editable"
     return install_req_from_editable(
         link.url,
+        user_supplied=template.user_supplied,
         comes_from=template.comes_from,
         use_pep517=template.use_pep517,
         isolated=template.isolated,
@@ -91,6 +93,7 @@ def make_install_req_from_dist(dist, template):
         line = "{}=={}".format(project_name, dist.parsed_version)
     ireq = install_req_from_line(
         line,
+        user_supplied=template.user_supplied,
         comes_from=template.comes_from,
         use_pep517=template.use_pep517,
         isolated=template.isolated,

src/pip/_internal/resolution/resolvelib/resolver.py

@@ -116,7 +116,7 @@ def resolve(self, root_reqs, check_supported_wheels):
                 else:
                     constraints[name] = req.specifier
             else:
-                if req.is_direct and req.name:
+                if req.user_supplied and req.name:
                     user_requested.add(canonicalize_name(req.name))
                 r = self.factory.make_requirement_from_install_req(
                     req, requested_extras=(),

tests/functional/test_install_requested.py

@@ -0,0 +1,94 @@
+def _assert_requested_present(script, result, name, version):
+    dist_info = script.site_packages / name + "-" + version + ".dist-info"
+    requested = dist_info / "REQUESTED"
+    assert dist_info in result.files_created
+    assert requested in result.files_created
+
+
+def _assert_requested_absent(script, result, name, version):
+    dist_info = script.site_packages / name + "-" + version + ".dist-info"
+    requested = dist_info / "REQUESTED"
+    assert dist_info in result.files_created
+    assert requested not in result.files_created
+
+
+def test_install_requested_basic(script, data, with_wheel):
+    result = script.pip(
+        "install", "--no-index", "-f", data.find_links, "require_simple"
+    )
+    _assert_requested_present(script, result, "require_simple", "1.0")
+    # dependency is not REQUESTED
+    _assert_requested_absent(script, result, "simple", "3.0")
+
+
+def test_install_requested_requirements(script, data, with_wheel):
+    script.scratch_path.joinpath("requirements.txt").write_text(
+        "require_simple\n"
+    )
+    result = script.pip(
+        "install",
+        "--no-index",
+        "-f",
+        data.find_links,
+        "-r",
+        script.scratch_path / "requirements.txt",
+    )
+    _assert_requested_present(script, result, "require_simple", "1.0")
+    _assert_requested_absent(script, result, "simple", "3.0")
+
+
+def test_install_requested_dep_in_requirements(script, data, with_wheel):
+    script.scratch_path.joinpath("requirements.txt").write_text(
+        "require_simple\nsimple<3\n"
+    )
+    result = script.pip(
+        "install",
+        "--no-index",
+        "-f",
+        data.find_links,
+        "-r",
+        script.scratch_path / "requirements.txt",
+    )
+    _assert_requested_present(script, result, "require_simple", "1.0")
+    # simple must have REQUESTED because it is in requirements.txt
+    _assert_requested_present(script, result, "simple", "2.0")
+
+
+def test_install_requested_reqs_and_constraints(script, data, with_wheel):
+    script.scratch_path.joinpath("requirements.txt").write_text(
+        "require_simple\n"
+    )
+    script.scratch_path.joinpath("constraints.txt").write_text("simple<3\n")
+    result = script.pip(
+        "install",
+        "--no-index",
+        "-f",
+        data.find_links,
+        "-r",
+        script.scratch_path / "requirements.txt",
+        "-c",
+        script.scratch_path / "constraints.txt",
+    )
+    _assert_requested_present(script, result, "require_simple", "1.0")
+    # simple must not have REQUESTED because it is merely a constraint
+    _assert_requested_absent(script, result, "simple", "2.0")
+
+
+def test_install_requested_in_reqs_and_constraints(script, data, with_wheel):
+    script.scratch_path.joinpath("requirements.txt").write_text(
+        "require_simple\nsimple\n"
+    )
+    script.scratch_path.joinpath("constraints.txt").write_text("simple<3\n")
+    result = script.pip(
+        "install",
+        "--no-index",
+        "-f",
+        data.find_links,
+        "-r",
+        script.scratch_path / "requirements.txt",
+        "-c",
+        script.scratch_path / "constraints.txt",
+    )
+    _assert_requested_present(script, result, "require_simple", "1.0")
+    # simple must have REQUESTED because it is in requirements.txt
+    _assert_requested_present(script, result, "simple", "2.0")

tests/unit/test_req.py

@@ -56,7 +56,7 @@ def get_processed_req_from_line(line, fname='file', lineno=1):
     parsed_req = handle_requirement_line(parsed_line)
     assert parsed_req is not None
     req = install_req_from_parsed_requirement(parsed_req)
-    req.is_direct = True
+    req.user_supplied = True
     return req
 
 
@@ -109,7 +109,7 @@ def test_no_reuse_existing_build_dir(self, data):
             pass
         reqset = RequirementSet()
         req = install_req_from_line('simple')
-        req.is_direct = True
+        req.user_supplied = True
         reqset.add_requirement(req)
         finder = make_test_finder(find_links=[data.find_links])
         with self._basic_resolver(finder) as resolver:
@@ -133,7 +133,7 @@ def test_environment_marker_extras(self, data):
         req = install_req_from_editable(
             data.packages.joinpath("LocalEnvironMarker")
         )
-        req.is_direct = True
+        req.user_supplied = True
         reqset.add_requirement(req)
         finder = make_test_finder(find_links=[data.find_links])
         with self._basic_resolver(finder) as resolver:
@@ -626,10 +626,10 @@ def test_exclusive_environment_markers():
     """Make sure RequirementSet accepts several excluding env markers"""
     eq36 = install_req_from_line(
         "Django>=1.6.10,<1.7 ; python_version == '3.6'")
-    eq36.is_direct = True
+    eq36.user_supplied = True
     ne36 = install_req_from_line(
         "Django>=1.6.10,<1.8 ; python_version != '3.6'")
-    ne36.is_direct = True
+    ne36.user_supplied = True
 
     req_set = RequirementSet()
     req_set.add_requirement(eq36)