diff --git a/vulnerabilities/pipelines/v2_improvers/flag_ghost_packages.py b/vulnerabilities/pipelines/v2_improvers/flag_ghost_packages.py index 647f80871..d712a4094 100644 --- a/vulnerabilities/pipelines/v2_improvers/flag_ghost_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/flag_ghost_packages.py @@ -10,16 +10,29 @@ import logging from itertools import groupby from traceback import format_exc as traceback_format_exc +from typing import NamedTuple from aboutcode.pipeline import LoopProgress from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS from fetchcode.package_versions import versions from packageurl import PackageURL +from requests import HTTPError from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline +class PackageNotFoundError(Exception): + """The package-level registry lookup returned HTTP 404.""" + + +class GhostPackageCounts(NamedTuple): + """Count missing package identities separately from individual missing versions.""" + + packages: int = 0 + versions: int = 0 + + class FlagGhostPackagePipeline(VulnerableCodePipeline): """Detect and flag packages that do not exist upstream.""" @@ -37,7 +50,7 @@ def flag_ghost_packages(self): def detect_and_flag_ghost_packages(logger=None): - """Check if packages are available upstream. If not, mark them as ghost package.""" + """Flag unavailable packages and versions, reporting separate totals for each.""" interesting_packages_qs = ( PackageV2.objects.order_by("type", "namespace", "name") .filter(type__in=FETCHCODE_SUPPORTED_ECOSYSTEMS) @@ -57,48 +70,91 @@ def detect_and_flag_ghost_packages(logger=None): ) ghost_package_count = 0 + ghost_version_count = 0 progress = LoopProgress(total_iterations=distinct_packages_count, logger=logger) for type_namespace_name, packages in progress.iter(grouped_packages): - ghost_package_count += flag_ghost_packages( + counts = flag_ghost_packages( base_purl=PackageURL(*type_namespace_name), packages=packages, logger=logger, ) + ghost_package_count += counts.packages + ghost_version_count += counts.versions if logger: - logger(f"Successfully flagged {ghost_package_count:,d} ghost Packages") + logger( + f"Successfully flagged {ghost_package_count:,d} ghost packages " + f"and {ghost_version_count:,d} ghost package versions" + ) def flag_ghost_packages(base_purl, packages, logger=None): """ - Check if `packages` are available upstream. - If not, update `is_ghost` to `True`. - Return the number of packages flagged as ghost. + Return separate counts for missing package names and missing versions. + + A missing package is reported once for its base PURL, while missing versions + of an existing package are reported individually for PurlDB verification. + Both keep ``is_ghost=True`` so unavailable versions cannot be suggested as fixes. """ - known_versions = get_versions(purl=base_purl, logger=logger) + try: + known_versions = get_versions(purl=base_purl, logger=logger) + except PackageNotFoundError: + flagged = False + for pkg in packages: + pkg.is_ghost = True + pkg.save() + flagged = True + + if flagged and logger: + logger(f"Flagging ghost package {base_purl!s}", level=logging.WARNING) + return GhostPackageCounts(packages=int(flagged)) + # Skip if encounter error while fetching known versions if known_versions is None: - return 0 + return GhostPackageCounts() - ghost_packages = 0 + ghost_versions = 0 for pkg in packages: - pkg.is_ghost = False - if pkg.version.lstrip("vV") not in known_versions: - pkg.is_ghost = True - ghost_packages += 1 + pkg.is_ghost = bool(pkg.version) and pkg.version.lstrip("vV") not in known_versions + if pkg.is_ghost: + ghost_versions += 1 if logger: - logger(f"Flagging ghost package {pkg.purl!s}", level=logging.DEBUG) + logger( + f"Flagging ghost package version {pkg.purl!s} for PurlDB verification", + level=logging.INFO, + ) pkg.save() - return ghost_packages + return GhostPackageCounts(versions=ghost_versions) def get_versions(purl, logger=None): - """Return set of known versions for the given purl.""" + """ + Return known versions, None on lookup failure, or raise PackageNotFoundError + when a package-level registry lookup returns HTTP 404. + + Empty release lists do not establish that the package itself is missing. + Go and GitHub lookups can fail for nested paths, version metadata or access + permissions, so their 404s are kept as inconclusive lookup failures. + """ + known_versions = set() try: - return {v.value.lstrip("vV") for v in versions(str(purl))} + if isinstance(purl, str): + purl = PackageURL.from_string(purl) + for version in versions(str(purl)): + known_versions.add(version.value.lstrip("vV")) + return known_versions except Exception as e: + if ( + isinstance(e, HTTPError) + and e.response is not None + and e.response.status_code == 404 + and not known_versions + and purl.type not in ("golang", "github") + ): + raise PackageNotFoundError(str(purl)) from e + if logger: logger( f"Error while fetching known versions for {purl!s}: {e!r} \n {traceback_format_exc()}", diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_flag_ghost_packages_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_flag_ghost_packages_v2.py index d082fdc3a..41c026a51 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_flag_ghost_packages_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_flag_ghost_packages_v2.py @@ -7,20 +7,28 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import logging +from unittest.mock import Mock from unittest.mock import patch import pytest +from fetchcode.package_versions import PackageVersion from packageurl import PackageURL +from requests import HTTPError +from requests import Response +from requests import Timeout from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines.v2_improvers.flag_ghost_packages import PackageNotFoundError from vulnerabilities.pipelines.v2_improvers.flag_ghost_packages import ( detect_and_flag_ghost_packages, ) from vulnerabilities.pipelines.v2_improvers.flag_ghost_packages import flag_ghost_packages +from vulnerabilities.pipelines.v2_improvers.flag_ghost_packages import get_versions @pytest.mark.django_db -def test_flag_ghost_package_marked_correctly(): +def test_flag_ghost_package_version_marked_correctly(): pkg = PackageV2.objects.create( type="pypi", namespace=None, @@ -34,11 +42,17 @@ def test_flag_ghost_package_marked_correctly(): mock_get_versions.return_value = {"2.25.1", "2.26.0"} base_purl = PackageURL(type="pypi", namespace=None, name="requests") - ghost_count = flag_ghost_packages(base_purl, [pkg]) + logger = Mock() + counts = flag_ghost_packages(base_purl, [pkg], logger=logger) pkg.refresh_from_db() - assert ghost_count == 1 assert pkg.is_ghost is True + logger.assert_called_once_with( + "Flagging ghost package version pkg:pypi/requests@999.999.999 for PurlDB verification", + level=logging.INFO, + ) + assert counts.packages == 0 + assert counts.versions == 1 @pytest.mark.django_db @@ -56,10 +70,11 @@ def test_flag_non_ghost_package_not_marked(): mock_get_versions.return_value = {"2.25.1", "2.26.0"} base_purl = PackageURL(type="pypi", namespace=None, name="requests") - ghost_count = flag_ghost_packages(base_purl, [pkg]) + counts = flag_ghost_packages(base_purl, [pkg]) pkg.refresh_from_db() - assert ghost_count == 0 + assert counts.packages == 0 + assert counts.versions == 0 assert pkg.is_ghost is False @@ -78,34 +93,183 @@ def test_flag_ghost_packages_gracefully_handles_version_fetch_failure(): mock_get_versions.return_value = None base_purl = PackageURL(type="pypi", namespace=None, name="some-lib") - ghost_count = flag_ghost_packages(base_purl, [pkg]) + counts = flag_ghost_packages(base_purl, [pkg]) pkg.refresh_from_db() - assert ghost_count == 0 + assert counts.packages == 0 + assert counts.versions == 0 assert pkg.is_ghost is False @pytest.mark.django_db def test_detect_and_flag_ghost_packages(monkeypatch): ghost_pkg = PackageV2.objects.create(type="pypi", name="fakepkg", version="9.9.9") + missing_pkg = PackageV2.objects.from_purl("pkg:pypi/missing@1.0.0") + missing_pkg2 = PackageV2.objects.from_purl("pkg:pypi/missing@2.0.0") real_pkg = PackageV2.objects.create(type="pypi", name="realpkg", version="1.0.0") - def fake_versions(purl, logger=None): - if purl.name == "realpkg": - return {"1.0.0"} - if purl.name == "fakepkg": - return {"0.1.0", "0.2.0"} - return set() + def fake_versions(purl): + if purl == "pkg:pypi/realpkg": + return [PackageVersion(value="1.0.0")] + if purl == "pkg:pypi/fakepkg": + return [PackageVersion(value="0.1.0"), PackageVersion(value="0.2.0")] + if purl == "pkg:pypi/missing": + raise registry_error(404) + raise AssertionError(f"Unexpected registry lookup: {purl}") monkeypatch.setattr( - "vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.get_versions", + "vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.versions", fake_versions, ) - detect_and_flag_ghost_packages() + logger = Mock() + detect_and_flag_ghost_packages(logger=logger) ghost_pkg.refresh_from_db() + missing_pkg.refresh_from_db() + missing_pkg2.refresh_from_db() real_pkg.refresh_from_db() assert ghost_pkg.is_ghost is True + assert missing_pkg.is_ghost is True + assert missing_pkg2.is_ghost is True assert real_pkg.is_ghost is False + logger.assert_any_call("Successfully flagged 1 ghost packages and 1 ghost package versions") + + +def registry_error(status_code, url="https://pypi.org/pypi/missing/json"): + response = Response() + response.status_code = status_code + response.url = url + return HTTPError(response=response) + + +@pytest.mark.parametrize("purl", ["pkg:pypi/missing", PackageURL(type="pypi", name="missing")]) +@patch("vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.versions") +def test_package_level_404_is_distinct_from_lookup_failure(mock_versions, purl): + mock_versions.side_effect = registry_error(404) + logger = Mock() + + with pytest.raises(PackageNotFoundError): + get_versions(purl, logger=logger) + + logger.assert_not_called() + + +@pytest.mark.django_db +@patch("vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.versions") +def test_missing_package_is_reported_once_for_all_versions(mock_versions): + packages = [ + PackageV2.objects.from_purl("pkg:pypi/missing@1.0.0"), + PackageV2.objects.from_purl("pkg:pypi/missing@2.0.0"), + ] + mock_versions.side_effect = registry_error(404) + logger = Mock() + + counts = flag_ghost_packages( + PackageURL(type="pypi", name="missing"), iter(packages), logger=logger + ) + + for package in packages: + package.refresh_from_db() + assert package.is_ghost is True + assert counts.packages == 1 + assert counts.versions == 0 + logger.assert_called_once_with("Flagging ghost package pkg:pypi/missing", level=logging.WARNING) + + +@pytest.mark.django_db +@patch("vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.versions") +def test_empty_release_list_does_not_mean_missing_package(mock_versions): + package = PackageV2.objects.from_purl("pkg:pypi/empty@1.0.0") + mock_versions.return_value = [] + logger = Mock() + + counts = flag_ghost_packages(PackageURL(type="pypi", name="empty"), [package], logger=logger) + + package.refresh_from_db() + assert package.is_ghost is True + assert counts.packages == 0 + assert counts.versions == 1 + logger.assert_called_once_with( + "Flagging ghost package version pkg:pypi/empty@1.0.0 for PurlDB verification", + level=logging.INFO, + ) + + +@pytest.mark.django_db +@patch("vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.versions") +def test_unversioned_existing_package_is_not_a_ghost_version(mock_versions): + package = PackageV2.objects.from_purl("pkg:pypi/empty") + mock_versions.return_value = [] + logger = Mock() + + counts = flag_ghost_packages(PackageURL(type="pypi", name="empty"), [package], logger=logger) + + package.refresh_from_db() + assert package.is_ghost is False + assert counts.packages == counts.versions == 0 + logger.assert_not_called() + + +@pytest.mark.django_db +@pytest.mark.parametrize("version", ["1.0.0", "v1.0.0", "V1.0.0"]) +@patch("vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.versions") +def test_known_version_clears_previous_ghost_flag(mock_versions, version): + package = PackageV2.objects.from_purl(f"pkg:pypi/real@{version}") + package.is_ghost = True + package.save() + mock_versions.return_value = [PackageVersion(value="v1.0.0")] + + counts = flag_ghost_packages(PackageURL(type="pypi", name="real"), [package]) + + package.refresh_from_db() + assert package.is_ghost is False + assert counts.packages == counts.versions == 0 + + +@pytest.mark.django_db +@pytest.mark.parametrize("was_ghost", [False, True]) +@pytest.mark.parametrize( + "error", [registry_error(403), registry_error(429), registry_error(500), Timeout()] +) +@patch("vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.versions") +def test_lookup_errors_preserve_existing_flags(mock_versions, error, was_ghost): + package = PackageV2.objects.from_purl("pkg:pypi/real@1.0.0") + package.is_ghost = was_ghost + package.save() + mock_versions.side_effect = error + logger = Mock() + + counts = flag_ghost_packages(PackageURL(type="pypi", name="real"), [package], logger=logger) + + package.refresh_from_db() + assert package.is_ghost is was_ghost + assert counts.packages == counts.versions == 0 + assert logger.call_args.kwargs["level"] == logging.ERROR + + +@patch("vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.versions") +def test_partial_version_lookup_404_does_not_report_missing_package(mock_versions): + def partial_versions(): + yield PackageVersion(value="1.0.0") + raise registry_error(404) + + mock_versions.return_value = partial_versions() + logger = Mock() + + assert get_versions(PackageURL(type="pypi", name="real"), logger=logger) is None + assert logger.call_args.kwargs["level"] == logging.ERROR + + +@pytest.mark.parametrize("endpoint", ["list", "v1.0.0.info"]) +@patch("vulnerabilities.pipelines.v2_improvers.flag_ghost_packages.versions") +def test_go_lookup_404_is_not_proof_of_missing_package(mock_versions, endpoint): + mock_versions.side_effect = registry_error( + 404, url=f"https://proxy.golang.org/github.com/example/module/@v/{endpoint}" + ) + logger = Mock() + + purl = PackageURL.from_string("pkg:golang/github.com/example/module") + assert get_versions(purl, logger=logger) is None + assert logger.call_args.kwargs["level"] == logging.ERROR