diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4c3d9efb4..f52bf437a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Release notes ============= +Version v38.1.0 +--------------------- + +- Throttle UI to 15 requests per minute to avoid abuse and improve performance. +- Handle errors in unfurl_version_range pipeline. +- Remove Todo pipeline from v1 pipelines. +- Add openAPI documentation for Package and Advisory viewset. + Version v38.0.0 --------------------- diff --git a/setup.cfg b/setup.cfg index 5c8efc7dd..16dbe9b9a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = vulnerablecode -version = 38.0.0 +version = 38.1.0 license = Apache-2.0 AND CC-BY-SA-4.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index ffa5bd941..c17202f25 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -14,6 +14,7 @@ from django.db.models import OuterRef from django.db.models import Prefetch from django_filters import rest_framework as filters +from drf_spectacular.utils import extend_schema from packageurl import PackageURL from rest_framework import serializers from rest_framework import viewsets @@ -422,6 +423,7 @@ class PackageV3ViewSet(viewsets.GenericViewSet): filter_backends = [filters.DjangoFilterBackend] throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] + @extend_schema(request=PackageQuerySerializer) def create(self, request, *args, **kwargs): serializer = PackageQuerySerializer(data=request.data) serializer.is_valid(raise_exception=True) @@ -528,8 +530,9 @@ class AdvisoryV3ViewSet(viewsets.GenericViewSet): filter_backends = [filters.DjangoFilterBackend] throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] + @extend_schema(request=AdvisoryQuerySerializer) def create(self, request, *args, **kwargs): - serializer = PackageQuerySerializer(data=request.data) + serializer = AdvisoryQuerySerializer(data=request.data) serializer.is_valid(raise_exception=True) purls = serializer.validated_data["purls"] diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 594021092..439e69731 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -21,7 +21,6 @@ from vulnerabilities.importers import github_osv from vulnerabilities.importers import istio from vulnerabilities.importers import mozilla -from vulnerabilities.importers import openssl from vulnerabilities.importers import oss_fuzz from vulnerabilities.importers import postgresql from vulnerabilities.importers import project_kb_msr2019 @@ -38,7 +37,6 @@ from vulnerabilities.pipelines import gitlab_importer from vulnerabilities.pipelines import nginx_importer from vulnerabilities.pipelines import npm_importer -from vulnerabilities.pipelines import nvd_importer from vulnerabilities.pipelines import pypa_importer from vulnerabilities.pipelines import pysec_importer from vulnerabilities.pipelines.v2_importers import alpine_linux_importer as alpine_linux_importer_v2 @@ -118,7 +116,6 @@ retiredotnet_importer_v2.RetireDotnetImporterPipeline, ubuntu_osv_importer_v2.UbuntuOSVImporterPipeline, alpine_linux_importer_v2.AlpineLinuxImporterPipeline, - nvd_importer.NVDImporterPipeline, github_importer.GitHubAPIImporterPipeline, gitlab_importer.GitLabImporterPipeline, github_osv.GithubOSVImporter, @@ -136,7 +133,6 @@ alpine_linux_importer.AlpineLinuxImporterPipeline, ruby.RubyImporter, apache_kafka.ApacheKafkaImporter, - openssl.OpensslImporter, openssl_importer_v2.OpenSSLImporterPipeline, redhat.RedhatImporter, archlinux.ArchlinuxImporter, diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index d55ecafdb..11fa5126a 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -10,7 +10,6 @@ from vulnerabilities.improvers import valid_versions from vulnerabilities.improvers import vulnerability_status from vulnerabilities.pipelines import add_cvss31_to_CVEs -from vulnerabilities.pipelines import compute_advisory_todo from vulnerabilities.pipelines import compute_package_risk from vulnerabilities.pipelines import compute_package_version_rank from vulnerabilities.pipelines import enhance_with_exploitdb @@ -70,7 +69,6 @@ compute_package_risk_v2.ComputePackageRiskPipeline, compute_version_rank_v2.ComputeVersionRankPipeline, unfurl_version_range_v2.UnfurlVersionRangePipeline, - compute_advisory_todo.ComputeToDo, collect_ssvc_trees.CollectSSVCPipeline, relate_severities.RelateSeveritiesPipeline, group_advisories_for_packages.GroupAdvisoriesForPackages, diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index 1d603b88a..f18f43fbf 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -52,7 +52,7 @@ def unfurl_version_range(self): if purl.type not in RANGE_CLASS_BY_SCHEMES: continue - versions = get_purl_versions(purl, cached_versions) + versions = get_purl_versions(purl, cached_versions) or [] affected_purls = get_affected_purls( versions=versions, affecting_vers=impact.affecting_vers, @@ -79,6 +79,8 @@ def get_affected_purls(versions, affecting_vers, base_purl, logger): version_class = affecting_version_range.version_class try: + if not versions: + return [] versions = [version_class(v) for v in versions] except Exception as e: logger( @@ -107,8 +109,10 @@ def get_affected_purls(versions, affecting_vers, base_purl, logger): def get_purl_versions(purl, cached_versions): if not purl in cached_versions: - cached_versions[purl] = get_versions(purl) - return cached_versions[purl] + purls = get_versions(purl) + if purls is not None: + cached_versions[purl] = purls + return cached_versions.get(purl) or [] def bulk_create_with_m2m(purls, impact, relation, logger): diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index 471e0bf43..3111ef738 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -11,8 +11,10 @@ import time import pytest +from django.core.cache import cache from django.test import Client from django.test import TestCase +from django.urls import reverse from packageurl import PackageURL from univers import versions @@ -330,3 +332,33 @@ def test_aggregate_fixed_and_affected_packages(self): end_time = time.time() assert end_time - start_time < 0.05 self.assertEqual(response.status_code, 200) + + +class ThrottleTestCase(TestCase): + def setUp(self): + self.client = Client() + cache.clear() + + def test_throttle_after_15_requests(self): + url = reverse("home") + + responses = [] + + for i in range(16): + response = self.client.get( + url, + HTTP_USER_AGENT="test-agent", + ) + responses.append(response.status_code) + + assert all(code == 200 for code in responses[:15]) + + assert responses[15] == 429 + + url = reverse("package_search") + + response = self.client.get( + url, + HTTP_USER_AGENT="test-agent", + ) + assert response.status_code == 429 diff --git a/vulnerabilities/throttling.py b/vulnerabilities/throttling.py index e14c1a1c0..c97b2c89f 100644 --- a/vulnerabilities/throttling.py +++ b/vulnerabilities/throttling.py @@ -51,6 +51,19 @@ def get_throttle_rate(self, tier): raise ImproperlyConfigured(msg) +class AnonUserUIThrottle(UserRateThrottle): + scope = "ui" + + def allow_request(self, request, view): + self.rate = self.THROTTLE_RATES.get("ui") + self.num_requests, self.duration = self.parse_rate(self.rate) + return super().allow_request(request, view) + + def get_cache_key(self, request, view): + ident = self.get_ident(request) + return f"throttle_ui_{ident}" + + def throttled_exception_handler(exception, context): """ Return this response whenever a request has been throttled diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 4f9f396ea..b984fbb51 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -20,6 +20,7 @@ from django.db.models import Exists from django.db.models import OuterRef from django.db.models import Prefetch +from django.http import HttpResponse from django.http.response import Http404 from django.shortcuts import get_object_or_404 from django.shortcuts import redirect @@ -47,6 +48,7 @@ from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS +from vulnerabilities.throttling import AnonUserUIThrottle from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS from vulnerabilities.utils import get_advisories_from_groups from vulnerabilities.utils import merge_and_save_grouped_advisories @@ -56,7 +58,47 @@ PAGE_SIZE = 10 -class PackageSearch(ListView): +class VulnerableCodeView(View): + """ + Base ListView for VulnerableCode views that includes throttling. + """ + + throttle_classes = [AnonUserUIThrottle] + + def dispatch(self, request, *args, **kwargs): + throttle = AnonUserUIThrottle() + + if not throttle.allow_request(request, self): + return HttpResponse("Rate limit exceeded", status=429) + + return super().dispatch(request, *args, **kwargs) + + +class VulnerableCodeDetailView(DetailView, VulnerableCodeView): + """ + Base DetailView for VulnerableCode views that includes throttling. + """ + + pass + + +class VulnerableCodeListView(ListView, VulnerableCodeView): + """ + Base ListView for VulnerableCode views that includes throttling. + """ + + pass + + +class VulnerableCodeCreateView(generic.CreateView, VulnerableCodeView): + """ + Base CreateView for VulnerableCode views that includes throttling. + """ + + pass + + +class PackageSearch(VulnerableCodeListView): model = models.Package template_name = "packages.html" ordering = ["type", "namespace", "name", "version"] @@ -84,7 +126,7 @@ def get_queryset(self, query=None): ) -class VulnerabilitySearch(ListView): +class VulnerabilitySearch(VulnerableCodeListView): model = models.Vulnerability template_name = "vulnerabilities.html" ordering = ["vulnerability_id"] @@ -102,7 +144,7 @@ def get_queryset(self, query=None): return self.model.objects.search(query=query).with_package_counts() -class PackageDetails(DetailView): +class PackageDetails(VulnerableCodeDetailView): model = models.Package template_name = "package_details.html" slug_url_kwarg = "purl" @@ -143,7 +185,7 @@ def get_object(self, queryset=None): return package -class PackageSearchV2(ListView): +class PackageSearchV2(VulnerableCodeListView): model = models.PackageV2 template_name = "packages_v2.html" ordering = ["type", "namespace", "name", "version"] @@ -166,7 +208,7 @@ def get_queryset(self, query=None): return self.model.objects.search(query).prefetch_related().with_is_vulnerable() -class AffectedByAdvisoriesListView(ListView): +class AffectedByAdvisoriesListView(VulnerableCodeListView): model = models.AdvisoryV2 template_name = "affected_by_advisories.html" paginate_by = PAGE_SIZE @@ -187,7 +229,7 @@ def get_queryset(self): ) -class FixingAdvisoriesListView(ListView): +class FixingAdvisoriesListView(VulnerableCodeListView): model = models.AdvisoryV2 template_name = "fixing_advisories.html" paginate_by = PAGE_SIZE @@ -201,7 +243,7 @@ def get_queryset(self): ) -class PackageV2Details(DetailView): +class PackageV2Details(VulnerableCodeDetailView): model = models.PackageV2 template_name = "package_details_v2.html" slug_url_kwarg = "purl" @@ -439,7 +481,7 @@ def get_fixed_package_details(package): return fixed_pkg_details -class VulnerabilityDetails(DetailView): +class VulnerabilityDetails(VulnerableCodeDetailView): model = models.Vulnerability template_name = "vulnerability_details.html" slug_url_kwarg = "vulnerability_id" @@ -543,7 +585,7 @@ def get_context_data(self, **kwargs): return context -class AdvisoryDetails(DetailView): +class AdvisoryDetails(VulnerableCodeDetailView): model = models.AdvisoryV2 template_name = "advisory_detail.html" slug_url_kwarg = "avid" @@ -717,7 +759,7 @@ def add_ssvc(ssvc): return context -class HomePage(View): +class HomePage(VulnerableCodeView): template_name = "index.html" def get(self, request): @@ -730,7 +772,7 @@ def get(self, request): return render(request=request, template_name=self.template_name, context=context) -class HomePageV2(View): +class HomePageV2(VulnerableCodeView): template_name = "index_v2.html" def get(self, request): @@ -770,7 +812,7 @@ def get(self, request): """ -class ApiUserCreateView(generic.CreateView): +class ApiUserCreateView(VulnerableCodeCreateView): model = models.ApiUser form_class = ApiUserCreationForm template_name = "api_user_creation_form.html" @@ -800,7 +842,7 @@ def get_success_url(self): return reverse_lazy("api_user_request") -class VulnerabilityPackagesDetails(DetailView): +class VulnerabilityPackagesDetails(VulnerableCodeDetailView): """ View to display all packages affected by or fixing a specific vulnerability. URL: /vulnerabilities/{vulnerability_id}/packages @@ -851,7 +893,7 @@ def get_context_data(self, **kwargs): return context -class AdvisoryPackagesDetails(DetailView): +class AdvisoryPackagesDetails(VulnerableCodeDetailView): """ View to display all packages affected by or fixing a specific vulnerability. URL: /advisories/{id}/packages @@ -902,7 +944,7 @@ def get_queryset(self): ) -class PipelineScheduleListView(ListView, FormMixin): +class PipelineScheduleListView(VulnerableCodeListView, FormMixin): model = PipelineSchedule context_object_name = "schedule_list" template_name = "pipeline_dashboard.html" @@ -926,7 +968,7 @@ def get_context_data(self, **kwargs): return context -class PipelineRunListView(ListView): +class PipelineRunListView(VulnerableCodeListView): model = PipelineRun context_object_name = "run_list" template_name = "pipeline_run_list.html" @@ -952,7 +994,7 @@ def get_context_data(self, **kwargs): return context -class PipelineRunDetailView(DetailView): +class PipelineRunDetailView(VulnerableCodeDetailView): model = PipelineRun template_name = "pipeline_run_details.html" context_object_name = "run" diff --git a/vulnerablecode/__init__.py b/vulnerablecode/__init__.py index 80b725801..4966e4c04 100644 --- a/vulnerablecode/__init__.py +++ b/vulnerablecode/__init__.py @@ -14,7 +14,7 @@ import git -__version__ = "38.0.0" +__version__ = "38.1.0" PROJECT_DIR = Path(__file__).resolve().parent diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index eaf2c1276..4c480cbc8 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -192,12 +192,14 @@ LOGOUT_REDIRECT_URL = "/" THROTTLE_RATE_ANON = env.str("THROTTLE_RATE_ANON", default="3600/hour") +THROTTLE_RATE_UI = env.str("THROTTLE_RATE_UI", default="15/minute") THROTTLE_RATE_USER_HIGH = env.str("THROTTLE_RATE_USER_HIGH", default="18000/hour") THROTTLE_RATE_USER_MEDIUM = env.str("THROTTLE_RATE_USER_MEDIUM", default="14400/hour") THROTTLE_RATE_USER_LOW = env.str("THROTTLE_RATE_USER_LOW", default="10800/hour") REST_FRAMEWORK_DEFAULT_THROTTLE_RATES = { "anon": THROTTLE_RATE_ANON, + "ui": THROTTLE_RATE_UI, "low": THROTTLE_RATE_USER_LOW, "medium": THROTTLE_RATE_USER_MEDIUM, "high": THROTTLE_RATE_USER_HIGH,