From 2ad391b4160261413997688516467338266b5216 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 4 Sep 2024 12:49:54 +0400 Subject: [PATCH] Refactoring progress #95 Signed-off-by: tdruez --- component_catalog/filters.py | 82 --------- component_catalog/models.py | 16 +- component_catalog/tests/test_admin.py | 2 +- component_catalog/tests/test_filters.py | 2 +- component_catalog/tests/test_models.py | 179 ------------------- component_catalog/tests/test_views.py | 68 +------- dje/templates/admin/base_site.html | 2 +- dje/templates/includes/navbar_header.html | 2 +- dje/tests/test_forms.py | 4 +- product_portfolio/views.py | 2 +- vulnerabilities/filters.py | 97 +++++++++++ vulnerabilities/models.py | 28 ++- vulnerabilities/tests/test_models.py | 203 ++++++++++++++++++++++ vulnerabilities/tests/test_views.py | 81 +++++++++ vulnerabilities/views.py | 2 +- 15 files changed, 417 insertions(+), 353 deletions(-) create mode 100644 vulnerabilities/filters.py create mode 100644 vulnerabilities/tests/test_models.py create mode 100644 vulnerabilities/tests/test_views.py diff --git a/component_catalog/filters.py b/component_catalog/filters.py index 158cb52d..83294143 100644 --- a/component_catalog/filters.py +++ b/component_catalog/filters.py @@ -8,7 +8,6 @@ from django import forms from django.contrib.admin.options import IncorrectLookupParameters -from django.db.models import F from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ @@ -23,12 +22,10 @@ from dje.filters import HasRelationFilter from dje.filters import MatchOrderedSearchFilter from dje.filters import RelatedLookupListFilter -from dje.filters import SearchFilter from dje.widgets import BootstrapSelectMultipleWidget from dje.widgets import DropDownRightWidget from dje.widgets import SortDropDownWidget from license_library.models import License -from vulnerabilities.models import Vulnerability class IsVulnerableFilter(HasRelationFilter): @@ -283,82 +280,3 @@ def show_created_date(self): @cached_property def show_last_modified_date(self): return not self.sort_value or self.has_sort_by("last_modified_date") - - -class NullsLastOrderingFilter(django_filters.OrderingFilter): - """ - A custom ordering filter that ensures null values are sorted last. - - When sorting by fields with potential null values, this filter modifies the - ordering to use Django's `nulls_last` clause for better handling of null values, - whether in ascending or descending order. - """ - - def filter(self, qs, value): - if not value: - return qs - - ordering = [] - for field in value: - if field.startswith("-"): - field_name = field[1:] - ordering.append(F(field_name).desc(nulls_last=True)) - else: - ordering.append(F(field).asc(nulls_last=True)) - - return qs.order_by(*ordering) - - -vulnerability_score_ranges = { - "low": (0.1, 3), - "medium": (4.0, 6.9), - "high": (7.0, 8.9), - "critical": (9.0, 10.0), -} - -SCORE_CHOICES = [ - (key, f"{key.capitalize()} ({value[0]} - {value[1]})") - for key, value in vulnerability_score_ranges.items() -] - - -class VulnerabilityFilterSet(DataspacedFilterSet): - q = SearchFilter( - label=_("Search"), - search_fields=["vulnerability_id", "aliases"], - ) - sort = NullsLastOrderingFilter( - label=_("Sort"), - fields=[ - "max_score", - "min_score", - "affected_products_count", - "affected_packages_count", - "fixed_packages_count", - "created_date", - "last_modified_date", - ], - widget=SortDropDownWidget, - ) - max_score = django_filters.ChoiceFilter( - choices=SCORE_CHOICES, - method="filter_by_score_range", - label="Score Range", - help_text="Select a score range to filter.", - ) - - class Meta: - model = Vulnerability - fields = [ - "q", - ] - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.filters["max_score"].extra["widget"] = DropDownRightWidget(anchor=self.anchor) - - def filter_by_score_range(self, queryset, name, value): - if value in vulnerability_score_ranges: - low, high = vulnerability_score_ranges[value] - return queryset.filter(max_score__gte=low, max_score__lte=high) - return queryset diff --git a/component_catalog/models.py b/component_catalog/models.py index 5816eb5a..3d0db089 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -80,6 +80,7 @@ from policy.models import SetPolicyFromLicenseMixin from policy.models import UsagePolicyMixin from vulnerabilities.models import AffectedByVulnerabilityMixin +from vulnerabilities.models import AffectedByVulnerabilityRelationship from workflow.models import RequestMixin logger = logging.getLogger("dje") @@ -2502,33 +2503,22 @@ def __str__(self): return f"{self.package} is under {self.license}." -# TODO: Review cascade -class PackageAffectedByVulnerability(DataspacedModel): +class PackageAffectedByVulnerability(AffectedByVulnerabilityRelationship): package = models.ForeignKey( to="component_catalog.Package", on_delete=models.CASCADE, ) - vulnerability = models.ForeignKey( - to="vulnerabilities.Vulnerability", - on_delete=models.PROTECT, - ) - class Meta: unique_together = (("package", "vulnerability"), ("dataspace", "uuid")) -class ComponentAffectedByVulnerability(DataspacedModel): +class ComponentAffectedByVulnerability(AffectedByVulnerabilityRelationship): component = models.ForeignKey( to="component_catalog.Component", on_delete=models.CASCADE, ) - vulnerability = models.ForeignKey( - to="vulnerabilities.Vulnerability", - on_delete=models.PROTECT, - ) - class Meta: unique_together = (("component", "vulnerability"), ("dataspace", "uuid")) diff --git a/component_catalog/tests/test_admin.py b/component_catalog/tests/test_admin.py index 5d70ace2..6d11ffe4 100644 --- a/component_catalog/tests/test_admin.py +++ b/component_catalog/tests/test_admin.py @@ -1980,7 +1980,7 @@ def test_package_changeform_filename_validation(self): self.assertEqual(200, response.status_code) self.assertEqual(errors, response.context_data["adminform"].form.errors) - @mock.patch("component_catalog.models.VulnerabilityMixin.fetch_vulnerabilities") + @mock.patch("vulnerabilities.models.AffectedByVulnerabilityMixin.fetch_vulnerabilities") def test_package_changeform_fetch_vulnerabilities(self, mock_fetch_vulnerabilities): mock_fetch_vulnerabilities.return_value = None self.dataspace1.enable_vulnerablecodedb_access = True diff --git a/component_catalog/tests/test_filters.py b/component_catalog/tests/test_filters.py index 5d10305e..f20940b4 100644 --- a/component_catalog/tests/test_filters.py +++ b/component_catalog/tests/test_filters.py @@ -14,7 +14,6 @@ from component_catalog.filters import ComponentFilterSet from component_catalog.filters import PackageFilterSet -from component_catalog.filters import VulnerabilityFilterSet from component_catalog.models import Component from component_catalog.models import ComponentKeyword from component_catalog.models import ComponentType @@ -26,6 +25,7 @@ from license_library.models import License from organization.models import Owner from policy.models import UsagePolicy +from vulnerabilities.filters import VulnerabilityFilterSet from vulnerabilities.tests import make_vulnerability diff --git a/component_catalog/tests/test_models.py b/component_catalog/tests/test_models.py index 0bc40df6..5f7b2cbb 100644 --- a/component_catalog/tests/test_models.py +++ b/component_catalog/tests/test_models.py @@ -38,12 +38,10 @@ from component_catalog.models import Package from component_catalog.models import PackageAlreadyExistsWarning from component_catalog.models import Subcomponent -from component_catalog.tests import make_component from component_catalog.tests import make_package from dejacode_toolkit import download from dejacode_toolkit.download import DataCollectionException from dejacode_toolkit.download import collect_package_data -from dejacode_toolkit.vulnerablecode import VulnerableCode from dje.copier import copy_object from dje.models import Dataspace from dje.models import History @@ -57,8 +55,6 @@ from license_library.models import LicenseTag from organization.models import Owner from product_portfolio.tests import make_product -from vulnerabilities.models import Vulnerability -from vulnerabilities.tests import make_vulnerability class ComponentCatalogModelsTestCase(TestCase): @@ -2585,178 +2581,3 @@ def test_vulnerability_mixin_is_vulnerable_property(self): package2 = make_package(self.dataspace) self.assertTrue(package1.is_vulnerable) self.assertFalse(package2.is_vulnerable) - - @mock.patch("dejacode_toolkit.vulnerablecode.VulnerableCode.request_get") - def test_vulnerability_mixin_get_entry_for_package(self, mock_request_get): - vulnerablecode = VulnerableCode(self.dataspace) - package1 = make_package(self.dataspace, package_url="pkg:composer/guzzlehttp/psr7@1.9.0") - response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" - mock_request_get.return_value = json.loads(response_file.read_text()) - - affected_by_vulnerabilities = package1.get_entry_for_package(vulnerablecode) - self.assertEqual(1, len(affected_by_vulnerabilities)) - self.assertEqual("VCID-j3au-usaz-aaag", affected_by_vulnerabilities[0]["vulnerability_id"]) - - @mock.patch("component_catalog.models.VulnerabilityMixin.get_entry_for_package") - @mock.patch("dejacode_toolkit.vulnerablecode.VulnerableCode.is_configured") - def test_vulnerability_mixin_get_entry_from_vulnerablecode( - self, mock_is_configured, mock_get_entry_for_package - ): - package1 = make_package(self.dataspace, package_url="pkg:pypi/idna@3.6") - self.assertIsNone(package1.get_entry_from_vulnerablecode()) - - mock_get_entry_for_package.return_value = None - self.dataspace.enable_vulnerablecodedb_access = True - self.dataspace.save() - mock_is_configured.return_value = True - package1.get_entry_from_vulnerablecode() - mock_get_entry_for_package.assert_called_once() - - @mock.patch("dejacode_toolkit.vulnerablecode.VulnerableCode.request_get") - @mock.patch("dejacode_toolkit.vulnerablecode.VulnerableCode.is_configured") - def test_vulnerability_mixin_fetch_vulnerabilities(self, mock_is_configured, mock_request_get): - mock_is_configured.return_value = True - self.dataspace.enable_vulnerablecodedb_access = True - self.dataspace.save() - response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" - mock_request_get.return_value = json.loads(response_file.read_text()) - - package1 = make_package(self.dataspace, package_url="pkg:pypi/idna@3.6") - package1.fetch_vulnerabilities() - - self.assertEqual(1, Vulnerability.objects.scope(self.dataspace).count()) - self.assertEqual(1, package1.affected_by_vulnerabilities.count()) - vulnerability = package1.affected_by_vulnerabilities.get() - self.assertEqual("VCID-j3au-usaz-aaag", vulnerability.vulnerability_id) - - def test_vulnerability_mixin_create_vulnerabilities(self): - response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" - response_json = json.loads(response_file.read_text()) - vulnerabilities_data = response_json["results"][0]["affected_by_vulnerabilities"] - - package1 = make_package(self.dataspace, package_url="pkg:pypi/idna@3.6") - package1.create_vulnerabilities(vulnerabilities_data) - - self.assertEqual(1, Vulnerability.objects.scope(self.dataspace).count()) - self.assertEqual(1, package1.affected_by_vulnerabilities.count()) - vulnerability = package1.affected_by_vulnerabilities.get() - self.assertEqual("VCID-j3au-usaz-aaag", vulnerability.vulnerability_id) - - def test_vulnerability_model_affected_packages_m2m(self): - package1 = make_package(self.dataspace) - vulnerablity1 = make_vulnerability(dataspace=self.dataspace, affecting=package1) - self.assertEqual(package1, vulnerablity1.affected_packages.get()) - self.assertEqual(vulnerablity1, package1.affected_by_vulnerabilities.get()) - - def test_vulnerability_model_add_affected(self): - vulnerablity1 = make_vulnerability(dataspace=self.dataspace) - package1 = make_package(self.dataspace) - package2 = make_package(self.dataspace) - vulnerablity1.add_affected(package1) - vulnerablity1.add_affected([package2]) - self.assertEqual(2, vulnerablity1.affected_packages.count()) - - vulnerablity2 = make_vulnerability(dataspace=self.dataspace) - component1 = make_component(self.dataspace) - vulnerablity2.add_affected([component1, package1]) - self.assertQuerySetEqual(vulnerablity2.affected_packages.all(), [package1]) - self.assertQuerySetEqual(vulnerablity2.affected_components.all(), [component1]) - - def test_vulnerability_model_fixed_packages_count_generated_field(self): - vulnerablity1 = make_vulnerability(dataspace=self.dataspace) - self.assertEqual(0, vulnerablity1.fixed_packages_count) - - vulnerablity1.fixed_packages = [ - {"purl": "pkg:pypi/gitpython@3.1.41", "is_vulnerable": True}, - {"purl": "pkg:pypi/gitpython@3.2", "is_vulnerable": False}, - ] - vulnerablity1.save() - vulnerablity1.refresh_from_db() - self.assertEqual(2, vulnerablity1.fixed_packages_count) - - def test_vulnerability_model_create_from_data(self): - package1 = make_package(self.dataspace) - vulnerability_data = { - "vulnerability_id": "VCID-q4q6-yfng-aaag", - "summary": "In Django 3.2 before 3.2.25, 4.2 before 4.2.11, and 5.0.", - "aliases": ["CVE-2024-27351", "GHSA-vm8q-m57g-pff3", "PYSEC-2024-47"], - "references": [ - { - "reference_url": "https://access.redhat.com/hydra/rest/" - "securitydata/cve/CVE-2024-27351.json", - "reference_id": "", - "scores": [ - { - "value": "7.5", - "scoring_system": "cvssv3", - "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - } - ], - }, - ], - } - - vulnerability1 = Vulnerability.create_from_data( - dataspace=self.dataspace, - data=vulnerability_data, - affecting=package1, - ) - self.assertEqual(vulnerability_data["vulnerability_id"], vulnerability1.vulnerability_id) - self.assertEqual(vulnerability_data["summary"], vulnerability1.summary) - self.assertEqual(vulnerability_data["aliases"], vulnerability1.aliases) - self.assertEqual(vulnerability_data["references"], vulnerability1.references) - self.assertEqual(7.5, vulnerability1.min_score) - self.assertEqual(7.5, vulnerability1.max_score) - self.assertQuerySetEqual(vulnerability1.affected_packages.all(), [package1]) - - def test_vulnerability_model_create_from_data_computed_scores(self): - response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" - json_data = json.loads(response_file.read_text()) - affected_by_vulnerabilities = json_data["results"][0]["affected_by_vulnerabilities"] - vulnerability1 = Vulnerability.create_from_data( - dataspace=self.dataspace, - data=affected_by_vulnerabilities[0], - ) - self.assertEqual(2.1, vulnerability1.min_score) - self.assertEqual(7.5, vulnerability1.max_score) - - def test_vulnerability_model_queryset_count_methods(self): - package1 = make_package(self.dataspace) - package2 = make_package(self.dataspace) - vulnerablity1 = make_vulnerability(dataspace=self.dataspace) - vulnerablity1.add_affected([package1, package2]) - make_product(self.dataspace, inventory=[package1, package2]) - - qs = ( - Vulnerability.objects.scope(self.dataspace) - .with_affected_products_count() - .with_affected_packages_count() - ) - self.assertEqual(2, qs[0].affected_packages_count) - self.assertEqual(1, qs[0].affected_products_count) - - def test_vulnerability_model_as_cyclonedx(self): - response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" - json_data = json.loads(response_file.read_text()) - affected_by_vulnerabilities = json_data["results"][0]["affected_by_vulnerabilities"] - vulnerability1 = Vulnerability.create_from_data( - dataspace=self.dataspace, - data=affected_by_vulnerabilities[0], - ) - package1 = make_package( - self.dataspace, - package_url="pkg:type/name@1.9.0", - uuid="dd0afd00-89bd-46d6-b1f0-57b553c44d32", - ) - - vulnerability1_as_cdx = vulnerability1.as_cyclonedx(affected_instances=[package1]) - as_dict = json.loads(vulnerability1_as_cdx.as_json()) - as_dict.pop("ratings", None) # The sorting is inconsistent - results = json.dumps(as_dict, indent=2) - - expected_location = self.data / "vulnerabilities" / "idna_3.6_as_cyclonedx.json" - # Uncomment to regen the expected results - # if True: - # expected_location.write_text(results) - - self.assertJSONEqual(results, expected_location.read_text()) diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index 34e8c31a..4b86d12c 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -39,8 +39,6 @@ from component_catalog.models import ComponentType from component_catalog.models import Package from component_catalog.models import Subcomponent -from component_catalog.tests import make_component -from component_catalog.tests import make_package from component_catalog.views import ComponentAddView from component_catalog.views import ComponentListView from component_catalog.views import PackageTabScanView @@ -49,7 +47,6 @@ from dejacode_toolkit.vulnerablecode import get_plain_purls from dje.copier import copy_object from dje.models import Dataspace -from dje.models import DataspaceConfiguration from dje.models import ExternalReference from dje.models import ExternalSource from dje.models import History @@ -70,7 +67,6 @@ from product_portfolio.models import ProductItemPurpose from product_portfolio.models import ProductPackage from product_portfolio.models import ProductRelationStatus -from vulnerabilities.models import Vulnerability from vulnerabilities.tests import make_vulnerability from workflow.models import Request from workflow.models import RequestTemplate @@ -1735,7 +1731,7 @@ def test_package_create_ajax_view(self): Package.objects.filter(download_url=collected_data["download_url"]).exists() ) - @mock.patch("component_catalog.models.VulnerabilityMixin.fetch_vulnerabilities") + @mock.patch("vulnerabilities.models.AffectedByVulnerabilityMixin.fetch_vulnerabilities") def test_package_create_ajax_view_fetch_vulnerabilities(self, mock_fetch_vulnerabilities): mock_fetch_vulnerabilities.return_value = None package_add_url = reverse("component_catalog:package_add_urls") @@ -3400,7 +3396,7 @@ def test_component_catalog_package_add_view_initial_data( } self.assertEqual(expected, response.context["form"].initial) - @mock.patch("component_catalog.models.VulnerabilityMixin.fetch_vulnerabilities") + @mock.patch("vulnerabilities.models.AffectedByVulnerabilityMixin.fetch_vulnerabilities") def test_component_catalog_package_add_view_fetch_vulnerabilities( self, mock_fetch_vulnerabilities ): @@ -4815,63 +4811,3 @@ def test_anonymous_user_cannot_access_reference_data(self): self.assertEqual(404, self.client.get(self.d1c2.get_absolute_url()).status_code) self.assertEqual(200, self.client.get(self.dmc1.get_absolute_url()).status_code) self.assertEqual(200, self.client.get(self.dmc2.get_absolute_url()).status_code) - - -class VulnerabilityViewsTestCase(TestCase): - def setUp(self): - self.dataspace = Dataspace.objects.create( - name="Dataspace", - enable_vulnerablecodedb_access=True, - ) - DataspaceConfiguration.objects.create( - dataspace=self.dataspace, - vulnerablecode_url="vulnerablecode_url/", - ) - self.super_user = create_superuser("super_user", self.dataspace) - - self.component1 = make_component(self.dataspace) - self.component2 = make_component(self.dataspace) - self.package1 = make_package(self.dataspace) - self.package2 = make_package(self.dataspace) - self.vulnerability_p1 = make_vulnerability(self.dataspace, affecting=self.component1) - self.vulnerability_c1 = make_vulnerability(self.dataspace, affecting=self.package1) - self.vulnerability1 = make_vulnerability(self.dataspace) - - def test_vulnerability_list_view_num_queries(self): - self.client.login(username=self.super_user.username, password="secret") - with self.assertNumQueries(8): - response = self.client.get(reverse("component_catalog:vulnerability_list")) - - vulnerability_count = Vulnerability.objects.count() - expected = f'{vulnerability_count} results' - self.assertContains(response, expected, html=True) - - def test_vulnerability_list_view_enable_vulnerablecodedb_access(self): - self.client.login(username=self.super_user.username, password="secret") - vulnerability_list_url = reverse("component_catalog:vulnerability_list") - response = self.client.get(vulnerability_list_url) - self.assertEqual(200, response.status_code) - vulnerability_header_link = ( - f'' - ) - self.assertContains(response, vulnerability_header_link) - - self.dataspace.enable_vulnerablecodedb_access = False - self.dataspace.save() - response = self.client.get(reverse("component_catalog:vulnerability_list")) - self.assertEqual(404, response.status_code) - - response = self.client.get(reverse("component_catalog:package_list")) - self.assertNotContains(response, vulnerability_header_link) - - def test_vulnerability_list_view_vulnerability_id_link(self): - self.client.login(username=self.super_user.username, password="secret") - response = self.client.get(reverse("component_catalog:vulnerability_list")) - expected = f""" - - {self.vulnerability1.vulnerability_id} - - - """ - self.assertContains(response, expected, html=True) diff --git a/dje/templates/admin/base_site.html b/dje/templates/admin/base_site.html index ae0db44c..f413d611 100644 --- a/dje/templates/admin/base_site.html +++ b/dje/templates/admin/base_site.html @@ -42,7 +42,7 @@

{% url 'workflow:request_list' as request_list_url %} {% url 'component_catalog:scan_list' as scan_list_url %} {% url 'purldb:purldb_list' as purldb_list_url %} - {% url 'component_catalog:vulnerability_list' as vulnerability_list_url %} + {% url 'vulnerabilities:vulnerability_list' as vulnerability_list_url %} {% url 'api_v2:api-root' as api_root_url %} {% if report_list_url or request_list_url or api_root_url %}
  • diff --git a/dje/templates/includes/navbar_header.html b/dje/templates/includes/navbar_header.html index 44948a75..90ca1343 100644 --- a/dje/templates/includes/navbar_header.html +++ b/dje/templates/includes/navbar_header.html @@ -11,7 +11,7 @@ {% url 'workflow:request_list' as request_list_url %} {% url 'component_catalog:scan_list' as scan_list_url %} {% url 'purldb:purldb_list' as purldb_list_url %} -{% url 'component_catalog:vulnerability_list' as vulnerability_list_url %} +{% url 'vulnerabilities:vulnerability_list' as vulnerability_list_url %} {% url 'django_registration_register' as register_url %} {% url 'api_v2:api-root' as api_root_url %} {% url 'account_profile' as account_profile_url %} diff --git a/dje/tests/test_forms.py b/dje/tests/test_forms.py index 668c90ca..ed21e465 100644 --- a/dje/tests/test_forms.py +++ b/dje/tests/test_forms.py @@ -110,8 +110,8 @@ def test_tabs_permission_formset_load_perms(self): def test_copy_defaults_form_get_all_dataspaced_models(self): dataspaced_models = CopyDefaultsForm.get_all_dataspaced_models() - self.assertEqual(9, len(dataspaced_models)) - self.assertEqual(8, len(dataspaced_models.get("Component Catalog"))) + self.assertEqual(10, len(dataspaced_models)) + self.assertEqual(7, len(dataspaced_models.get("Component Catalog"))) self.assertIn("Subcomponent", str(dataspaced_models.get("Component Catalog"))) def test_copy_defaults_formset_serialize_perms(self): diff --git a/product_portfolio/views.py b/product_portfolio/views.py index 037da34b..68848905 100644 --- a/product_portfolio/views.py +++ b/product_portfolio/views.py @@ -55,7 +55,6 @@ from openpyxl.styles import NamedStyle from openpyxl.styles import Side -from component_catalog.filters import VulnerabilityFilterSet from component_catalog.forms import ComponentAjaxForm from component_catalog.license_expression_dje import build_licensing from component_catalog.license_expression_dje import parse_expression @@ -126,6 +125,7 @@ from product_portfolio.models import ProductDependency from product_portfolio.models import ProductPackage from product_portfolio.models import ScanCodeProject +from vulnerabilities.filters import VulnerabilityFilterSet from vulnerabilities.models import Vulnerability diff --git a/vulnerabilities/filters.py b/vulnerabilities/filters.py new file mode 100644 index 00000000..e1ae9e5a --- /dev/null +++ b/vulnerabilities/filters.py @@ -0,0 +1,97 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# DejaCode is a trademark of nexB Inc. +# SPDX-License-Identifier: AGPL-3.0-only +# See https://github.com/aboutcode-org/dejacode for support or download. +# See https://aboutcode.org for more information about AboutCode FOSS projects. +# + +from django.db.models import F +from django.utils.translation import gettext_lazy as _ + +import django_filters + +from dje.filters import DataspacedFilterSet +from dje.filters import SearchFilter +from dje.widgets import DropDownRightWidget +from dje.widgets import SortDropDownWidget +from vulnerabilities.models import Vulnerability + + +class NullsLastOrderingFilter(django_filters.OrderingFilter): + """ + A custom ordering filter that ensures null values are sorted last. + + When sorting by fields with potential null values, this filter modifies the + ordering to use Django's `nulls_last` clause for better handling of null values, + whether in ascending or descending order. + """ + + def filter(self, qs, value): + if not value: + return qs + + ordering = [] + for field in value: + if field.startswith("-"): + field_name = field[1:] + ordering.append(F(field_name).desc(nulls_last=True)) + else: + ordering.append(F(field).asc(nulls_last=True)) + + return qs.order_by(*ordering) + + +vulnerability_score_ranges = { + "low": (0.1, 3), + "medium": (4.0, 6.9), + "high": (7.0, 8.9), + "critical": (9.0, 10.0), +} + +SCORE_CHOICES = [ + (key, f"{key.capitalize()} ({value[0]} - {value[1]})") + for key, value in vulnerability_score_ranges.items() +] + + +class VulnerabilityFilterSet(DataspacedFilterSet): + q = SearchFilter( + label=_("Search"), + search_fields=["vulnerability_id", "aliases"], + ) + sort = NullsLastOrderingFilter( + label=_("Sort"), + fields=[ + "max_score", + "min_score", + "affected_products_count", + "affected_packages_count", + "fixed_packages_count", + "created_date", + "last_modified_date", + ], + widget=SortDropDownWidget, + ) + max_score = django_filters.ChoiceFilter( + choices=SCORE_CHOICES, + method="filter_by_score_range", + label="Score Range", + help_text="Select a score range to filter.", + ) + + class Meta: + model = Vulnerability + fields = [ + "q", + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.filters["max_score"].extra["widget"] = DropDownRightWidget(anchor=self.anchor) + + def filter_by_score_range(self, queryset, name, value): + if value in vulnerability_score_ranges: + low, high = vulnerability_score_ranges[value] + return queryset.filter(max_score__gte=low, max_score__lte=high) + return queryset diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 8d91f3ba..441cca6f 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -127,17 +127,19 @@ def add_affected(self, instances): for instance in instances: if isinstance(instance, Package): - self.affected_packages.add(instance) + self.add_affected_packages([instance]) if isinstance(instance, Component): - self.affected_components.add(instance) + self.add_affected_components([instance]) def add_affected_packages(self, packages): """Assign the ``packages`` as affected to this vulnerability.""" - self.affected_packages.add(*packages) + through_defaults = {"dataspace_id": self.dataspace_id} + self.affected_packages.add(*packages, through_defaults=through_defaults) def add_affected_components(self, components): """Assign the ``components`` as affected to this vulnerability.""" - self.affected_components.add(*components) + through_defaults = {"dataspace_id": self.dataspace_id} + self.affected_components.add(*components, through_defaults=through_defaults) @staticmethod def range_to_values(self, range_str): @@ -254,6 +256,21 @@ def as_cyclonedx(self, affected_instances): ) +# TODO: Review cascade +class AffectedByVulnerabilityRelationship(DataspacedModel): + vulnerability = models.ForeignKey( + to="vulnerabilities.Vulnerability", + on_delete=models.PROTECT, + ) + + class Meta: + abstract = True + + def save(self, *args, **kwargs): + self.dataspace = self.vulnerability.dataspace + super().save(*args, **kwargs) + + class AffectedByVulnerabilityMixin(models.Model): """Add the `vulnerability` many to many field.""" @@ -332,4 +349,5 @@ def create_vulnerabilities(self, vulnerabilities_data): ) vulnerabilities.append(vulnerability) - self.affected_by_vulnerabilities.add(*vulnerabilities) + through_defaults = {"dataspace_id": self.dataspace_id} + self.affected_by_vulnerabilities.add(*vulnerabilities, through_defaults=through_defaults) diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py new file mode 100644 index 00000000..b9114b57 --- /dev/null +++ b/vulnerabilities/tests/test_models.py @@ -0,0 +1,203 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# DejaCode is a trademark of nexB Inc. +# SPDX-License-Identifier: AGPL-3.0-only +# See https://github.com/aboutcode-org/dejacode for support or download. +# See https://aboutcode.org for more information about AboutCode FOSS projects. +# + +import json +from pathlib import Path +from unittest import mock + +from django.test import TestCase + +from component_catalog.tests import make_component +from component_catalog.tests import make_package +from dejacode_toolkit.vulnerablecode import VulnerableCode +from dje.models import Dataspace +from product_portfolio.tests import make_product +from vulnerabilities.models import Vulnerability +from vulnerabilities.tests import make_vulnerability + + +class VulnerabilitiesFetchTestCase(TestCase): + data = Path(__file__).parent / "data" + + def setUp(self): + self.dataspace = Dataspace.objects.create(name="nexB") + + @mock.patch("dejacode_toolkit.vulnerablecode.VulnerableCode.request_get") + def test_vulnerability_mixin_get_entry_for_package(self, mock_request_get): + vulnerablecode = VulnerableCode(self.dataspace) + package1 = make_package(self.dataspace, package_url="pkg:composer/guzzlehttp/psr7@1.9.0") + response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" + mock_request_get.return_value = json.loads(response_file.read_text()) + + affected_by_vulnerabilities = package1.get_entry_for_package(vulnerablecode) + self.assertEqual(1, len(affected_by_vulnerabilities)) + self.assertEqual("VCID-j3au-usaz-aaag", affected_by_vulnerabilities[0]["vulnerability_id"]) + + @mock.patch("vulnerabilities.models.AffectedByVulnerabilityMixin.get_entry_for_package") + @mock.patch("dejacode_toolkit.vulnerablecode.VulnerableCode.is_configured") + def test_vulnerability_mixin_get_entry_from_vulnerablecode( + self, mock_is_configured, mock_get_entry_for_package + ): + package1 = make_package(self.dataspace, package_url="pkg:pypi/idna@3.6") + self.assertIsNone(package1.get_entry_from_vulnerablecode()) + + mock_get_entry_for_package.return_value = None + self.dataspace.enable_vulnerablecodedb_access = True + self.dataspace.save() + mock_is_configured.return_value = True + package1.get_entry_from_vulnerablecode() + mock_get_entry_for_package.assert_called_once() + + @mock.patch("dejacode_toolkit.vulnerablecode.VulnerableCode.request_get") + @mock.patch("dejacode_toolkit.vulnerablecode.VulnerableCode.is_configured") + def test_vulnerability_mixin_fetch_vulnerabilities(self, mock_is_configured, mock_request_get): + mock_is_configured.return_value = True + self.dataspace.enable_vulnerablecodedb_access = True + self.dataspace.save() + response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" + mock_request_get.return_value = json.loads(response_file.read_text()) + + package1 = make_package(self.dataspace, package_url="pkg:pypi/idna@3.6") + package1.fetch_vulnerabilities() + + self.assertEqual(1, Vulnerability.objects.scope(self.dataspace).count()) + self.assertEqual(1, package1.affected_by_vulnerabilities.count()) + vulnerability = package1.affected_by_vulnerabilities.get() + self.assertEqual("VCID-j3au-usaz-aaag", vulnerability.vulnerability_id) + + def test_vulnerability_mixin_create_vulnerabilities(self): + response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" + response_json = json.loads(response_file.read_text()) + vulnerabilities_data = response_json["results"][0]["affected_by_vulnerabilities"] + + package1 = make_package(self.dataspace, package_url="pkg:pypi/idna@3.6") + package1.create_vulnerabilities(vulnerabilities_data) + + self.assertEqual(1, Vulnerability.objects.scope(self.dataspace).count()) + self.assertEqual(1, package1.affected_by_vulnerabilities.count()) + vulnerability = package1.affected_by_vulnerabilities.get() + self.assertEqual("VCID-j3au-usaz-aaag", vulnerability.vulnerability_id) + + def test_vulnerability_model_affected_packages_m2m(self): + package1 = make_package(self.dataspace) + vulnerablity1 = make_vulnerability(dataspace=self.dataspace, affecting=package1) + self.assertEqual(package1, vulnerablity1.affected_packages.get()) + self.assertEqual(vulnerablity1, package1.affected_by_vulnerabilities.get()) + + def test_vulnerability_model_add_affected(self): + vulnerablity1 = make_vulnerability(dataspace=self.dataspace) + package1 = make_package(self.dataspace) + package2 = make_package(self.dataspace) + vulnerablity1.add_affected(package1) + vulnerablity1.add_affected([package2]) + self.assertEqual(2, vulnerablity1.affected_packages.count()) + + vulnerablity2 = make_vulnerability(dataspace=self.dataspace) + component1 = make_component(self.dataspace) + vulnerablity2.add_affected([component1, package1]) + self.assertQuerySetEqual(vulnerablity2.affected_packages.all(), [package1]) + self.assertQuerySetEqual(vulnerablity2.affected_components.all(), [component1]) + + def test_vulnerability_model_fixed_packages_count_generated_field(self): + vulnerablity1 = make_vulnerability(dataspace=self.dataspace) + self.assertEqual(0, vulnerablity1.fixed_packages_count) + + vulnerablity1.fixed_packages = [ + {"purl": "pkg:pypi/gitpython@3.1.41", "is_vulnerable": True}, + {"purl": "pkg:pypi/gitpython@3.2", "is_vulnerable": False}, + ] + vulnerablity1.save() + vulnerablity1.refresh_from_db() + self.assertEqual(2, vulnerablity1.fixed_packages_count) + + def test_vulnerability_model_create_from_data(self): + package1 = make_package(self.dataspace) + vulnerability_data = { + "vulnerability_id": "VCID-q4q6-yfng-aaag", + "summary": "In Django 3.2 before 3.2.25, 4.2 before 4.2.11, and 5.0.", + "aliases": ["CVE-2024-27351", "GHSA-vm8q-m57g-pff3", "PYSEC-2024-47"], + "references": [ + { + "reference_url": "https://access.redhat.com/hydra/rest/" + "securitydata/cve/CVE-2024-27351.json", + "reference_id": "", + "scores": [ + { + "value": "7.5", + "scoring_system": "cvssv3", + "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + } + ], + }, + ], + } + + vulnerability1 = Vulnerability.create_from_data( + dataspace=self.dataspace, + data=vulnerability_data, + affecting=package1, + ) + self.assertEqual(vulnerability_data["vulnerability_id"], vulnerability1.vulnerability_id) + self.assertEqual(vulnerability_data["summary"], vulnerability1.summary) + self.assertEqual(vulnerability_data["aliases"], vulnerability1.aliases) + self.assertEqual(vulnerability_data["references"], vulnerability1.references) + self.assertEqual(7.5, vulnerability1.min_score) + self.assertEqual(7.5, vulnerability1.max_score) + self.assertQuerySetEqual(vulnerability1.affected_packages.all(), [package1]) + + def test_vulnerability_model_create_from_data_computed_scores(self): + response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" + json_data = json.loads(response_file.read_text()) + affected_by_vulnerabilities = json_data["results"][0]["affected_by_vulnerabilities"] + vulnerability1 = Vulnerability.create_from_data( + dataspace=self.dataspace, + data=affected_by_vulnerabilities[0], + ) + self.assertEqual(2.1, vulnerability1.min_score) + self.assertEqual(7.5, vulnerability1.max_score) + + def test_vulnerability_model_queryset_count_methods(self): + package1 = make_package(self.dataspace) + package2 = make_package(self.dataspace) + vulnerablity1 = make_vulnerability(dataspace=self.dataspace) + vulnerablity1.add_affected([package1, package2]) + make_product(self.dataspace, inventory=[package1, package2]) + + qs = ( + Vulnerability.objects.scope(self.dataspace) + .with_affected_products_count() + .with_affected_packages_count() + ) + self.assertEqual(2, qs[0].affected_packages_count) + self.assertEqual(1, qs[0].affected_products_count) + + def test_vulnerability_model_as_cyclonedx(self): + response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" + json_data = json.loads(response_file.read_text()) + affected_by_vulnerabilities = json_data["results"][0]["affected_by_vulnerabilities"] + vulnerability1 = Vulnerability.create_from_data( + dataspace=self.dataspace, + data=affected_by_vulnerabilities[0], + ) + package1 = make_package( + self.dataspace, + package_url="pkg:type/name@1.9.0", + uuid="dd0afd00-89bd-46d6-b1f0-57b553c44d32", + ) + + vulnerability1_as_cdx = vulnerability1.as_cyclonedx(affected_instances=[package1]) + as_dict = json.loads(vulnerability1_as_cdx.as_json()) + as_dict.pop("ratings", None) # The sorting is inconsistent + results = json.dumps(as_dict, indent=2) + + expected_location = self.data / "vulnerabilities" / "idna_3.6_as_cyclonedx.json" + # Uncomment to regen the expected results + # if True: + # expected_location.write_text(results) + + self.assertJSONEqual(results, expected_location.read_text()) diff --git a/vulnerabilities/tests/test_views.py b/vulnerabilities/tests/test_views.py new file mode 100644 index 00000000..68eb19d8 --- /dev/null +++ b/vulnerabilities/tests/test_views.py @@ -0,0 +1,81 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# DejaCode is a trademark of nexB Inc. +# SPDX-License-Identifier: AGPL-3.0-only +# See https://github.com/aboutcode-org/dejacode for support or download. +# See https://aboutcode.org for more information about AboutCode FOSS projects. +# + +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.urls import reverse + +from component_catalog.tests import make_component +from component_catalog.tests import make_package +from dje.models import Dataspace +from dje.models import DataspaceConfiguration +from dje.tests import create_superuser +from vulnerabilities.models import Vulnerability +from vulnerabilities.tests import make_vulnerability + +User = get_user_model() + + +class VulnerabilityViewsTestCase(TestCase): + def setUp(self): + self.dataspace = Dataspace.objects.create( + name="Dataspace", + enable_vulnerablecodedb_access=True, + ) + DataspaceConfiguration.objects.create( + dataspace=self.dataspace, + vulnerablecode_url="vulnerablecode_url/", + ) + self.super_user = create_superuser("super_user", self.dataspace) + + self.component1 = make_component(self.dataspace) + self.component2 = make_component(self.dataspace) + self.package1 = make_package(self.dataspace) + self.package2 = make_package(self.dataspace) + self.vulnerability_p1 = make_vulnerability(self.dataspace, affecting=self.component1) + self.vulnerability_c1 = make_vulnerability(self.dataspace, affecting=self.package1) + self.vulnerability1 = make_vulnerability(self.dataspace) + + def test_vulnerability_list_view_num_queries(self): + self.client.login(username=self.super_user.username, password="secret") + with self.assertNumQueries(8): + response = self.client.get(reverse("vulnerabilities:vulnerability_list")) + + vulnerability_count = Vulnerability.objects.count() + expected = f'{vulnerability_count} results' + self.assertContains(response, expected, html=True) + + def test_vulnerability_list_view_enable_vulnerablecodedb_access(self): + self.client.login(username=self.super_user.username, password="secret") + vulnerability_list_url = reverse("vulnerabilities:vulnerability_list") + response = self.client.get(vulnerability_list_url) + self.assertEqual(200, response.status_code) + vulnerability_header_link = ( + f'' + ) + self.assertContains(response, vulnerability_header_link) + + self.dataspace.enable_vulnerablecodedb_access = False + self.dataspace.save() + response = self.client.get(reverse("vulnerabilities:vulnerability_list")) + self.assertEqual(404, response.status_code) + + response = self.client.get(reverse("component_catalog:package_list")) + self.assertNotContains(response, vulnerability_header_link) + + def test_vulnerability_list_view_vulnerability_id_link(self): + self.client.login(username=self.super_user.username, password="secret") + response = self.client.get(reverse("vulnerabilities:vulnerability_list")) + expected = f""" + + {self.vulnerability1.vulnerability_id} + + + """ + self.assertContains(response, expected, html=True) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 05d810f5..92320c67 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -11,10 +11,10 @@ from django.http import Http404 from django.utils.translation import gettext_lazy as _ -from component_catalog.filters import VulnerabilityFilterSet from dejacode_toolkit.vulnerablecode import VulnerableCode from dje.views import DataspacedFilterView from dje.views import Header +from vulnerabilities.filters import VulnerabilityFilterSet from vulnerabilities.models import Vulnerability