diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index 268419587..c6ec0f444 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -145,8 +145,6 @@ def get_aliases(record) -> [str]: aliases = [] if record.get("cve"): aliases.append("CVE-{}".format(record.get("cve"))) - if record.get("osvdb"): - aliases.append("OSV-{}".format(record.get("osvdb"))) if record.get("ghsa"): aliases.append("GHSA-{}".format(record.get("ghsa"))) return aliases diff --git a/vulnerabilities/migrations/0143_drop_osv_aliases.py b/vulnerabilities/migrations/0143_drop_osv_aliases.py new file mode 100644 index 000000000..8b8af74bb --- /dev/null +++ b/vulnerabilities/migrations/0143_drop_osv_aliases.py @@ -0,0 +1,156 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.db import migrations + +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.importer import PatchData +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.utils import compute_content_id_v2 +from vulnerabilities.utils import normalize_list +from vulnerabilities.utils import purl_to_dict + +""" +Drop legacy OSVDB-derived aliases imported from rubygem data sources. + +The rubysec advisory data contains legacy references to the defunct OSVDB +database in its `osvdb` field, and these were imported as `OSV-` +aliases. These are not public aliases and collide with the modern OSV.dev +namespace. Genuine OSV.dev identifiers have the form `OSV--` +and are preserved. + +See https://github.com/aboutcode-org/vulnerablecode/issues/2421 +""" + +OSVDB_DERIVED_ALIAS_REGEX = r"^OSV-\d+$" + + +def commit_patch_to_dict(patch): + return { + "vcs_url": patch.vcs_url, + "commit_hash": patch.commit_hash, + "patch_text": patch.patch_text, + "patch_checksum": patch.patch_checksum, + } + + +def to_affected_package_data(impact): + """Return `AffectedPackageV2` data from the impact.""" + return AffectedPackageV2.from_dict( + { + "package": purl_to_dict(impact.base_purl), + "affected_version_range": impact.affecting_vers, + "fixed_version_range": impact.fixed_vers, + "introduced_by_commit_patches": [ + commit_patch_to_dict(commit) + for commit in impact.introduced_by_package_commit_patches.all() + ], + "fixed_by_commit_patches": [ + commit_patch_to_dict(commit) + for commit in impact.fixed_by_package_commit_patches.all() + ], + } + ) + + +def to_patch_data(patch): + """Return `PatchData` from the Patch.""" + return PatchData.from_dict( + { + "patch_url": patch.patch_url, + "patch_text": patch.patch_text, + "patch_checksum": patch.patch_checksum, + } + ) + + +def to_reference_v2_data(ref): + return ReferenceV2.from_dict( + { + "reference_id": ref.reference_id, + "reference_type": ref.reference_type, + "url": ref.url, + } + ) + + +def to_vulnerability_severity_data(severity): + return VulnerabilitySeverity.from_dict( + { + "system": severity.scoring_system, + "value": severity.value, + "scoring_elements": severity.scoring_elements, + "published_at": severity.published_at, + "url": severity.url, + } + ) + + +def to_advisory_data(advisory): + return AdvisoryDataV2( + advisory_id=advisory.advisory_id, + aliases=normalize_list([item.alias for item in advisory.aliases.all()]), + summary=advisory.summary, + affected_packages=normalize_list( + [to_affected_package_data(impacted) for impacted in advisory.impacted_packages.all()] + ), + references=normalize_list([to_reference_v2_data(ref) for ref in advisory.references.all()]), + patches=normalize_list([to_patch_data(patch) for patch in advisory.patches.all()]), + date_published=advisory.date_published, + weaknesses=normalize_list([weak.cwe_id for weak in advisory.weaknesses.all()]), + severities=normalize_list( + [to_vulnerability_severity_data(sev) for sev in advisory.severities.all()] + ), + url=advisory.url, + ) + + +def drop_osv_aliases(apps, schema_editor): + AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias") + AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2") + Alias = apps.get_model("vulnerabilities", "Alias") + + # Identify affected advisories before deleting associations + advisory_ids = list( + AdvisoryV2.objects.filter(aliases__alias__iregex=OSVDB_DERIVED_ALIAS_REGEX) + .distinct() + .values_list("id", flat=True) + ) + + # Delete legacy OSV- (OSVDB-derived) aliases + AdvisoryAlias.objects.filter(alias__iregex=OSVDB_DERIVED_ALIAS_REGEX).delete() + Alias.objects.filter(alias__iregex=OSVDB_DERIVED_ALIAS_REGEX).delete() + + # Recompute unique_content_id for affected advisories + batch = [] + batch_size = 2000 + for advisory in AdvisoryV2.objects.filter(id__in=advisory_ids).iterator(chunk_size=1000): + advisory.unique_content_id = compute_content_id_v2(to_advisory_data(advisory)) + batch.append(advisory) + if len(batch) >= batch_size: + AdvisoryV2.objects.bulk_update(batch, ["unique_content_id"]) + batch.clear() + if batch: + AdvisoryV2.objects.bulk_update(batch, ["unique_content_id"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0142_advisoryv2_is_curation_advisoryv2_resolves_todos"), + ] + + operations = [ + migrations.RunPython( + drop_osv_aliases, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/vulnerabilities/pipelines/v2_importers/ruby_importer.py b/vulnerabilities/pipelines/v2_importers/ruby_importer.py index 5858ad00c..189dc5cea 100644 --- a/vulnerabilities/pipelines/v2_importers/ruby_importer.py +++ b/vulnerabilities/pipelines/v2_importers/ruby_importer.py @@ -202,8 +202,6 @@ def get_aliases(record) -> [str]: aliases = [] if record.get("cve"): aliases.append("CVE-{}".format(record.get("cve"))) - if record.get("osvdb"): - aliases.append("OSV-{}".format(record.get("osvdb"))) if record.get("ghsa"): aliases.append("GHSA-{}".format(record.get("ghsa"))) return aliases diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_ruby_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_ruby_importer_v2.py index 6f7c6644b..23da5eafc 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_ruby_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_ruby_importer_v2.py @@ -37,3 +37,39 @@ def test_ruby_advisories_per_file(yml_file): expected_file = yml_file.with_name(yml_file.stem + "-expected.json") util_tests.check_results_against_json(result, expected_file) + + +from vulnerabilities.pipelines.v2_importers.ruby_importer import get_aliases + + +def test_get_aliases_drops_osvdb_alias(): + record = { + "cve": "2020-0001", + "ghsa": "xxxx-yyyy-zzzz", + "osvdb": 12345, + } + aliases = get_aliases(record) + assert aliases == ["CVE-2020-0001", "GHSA-xxxx-yyyy-zzzz"] + assert "OSV-12345" not in aliases + + +def test_get_aliases_only_osvdb(): + record = { + "osvdb": 65123, + } + assert get_aliases(record) == [] + + +def test_get_aliases_no_osvdb(): + record = { + "cve": "2021-1234", + } + assert get_aliases(record) == ["CVE-2021-1234"] + + +def test_get_aliases_osvdb_and_ghsa_only(): + record = { + "ghsa": "xxxx-yyyy-zzzz", + "osvdb": 12345, + } + assert get_aliases(record) == ["GHSA-xxxx-yyyy-zzzz"] diff --git a/vulnerabilities/tests/test_data/ruby/CVE-2010-1330-expected.json b/vulnerabilities/tests/test_data/ruby/CVE-2010-1330-expected.json index fa77e85a8..5cc203812 100644 --- a/vulnerabilities/tests/test_data/ruby/CVE-2010-1330-expected.json +++ b/vulnerabilities/tests/test_data/ruby/CVE-2010-1330-expected.json @@ -1,7 +1,6 @@ { "aliases": [ - "CVE-2010-1330", - "OSV-77297" + "CVE-2010-1330" ], "summary": "CVE-2010-1330 jruby: XSS in the regular expression engine when processing invalid UTF-8 byte sequences\nThe regular expression engine in JRuby before 1.4.1, when $KCODE is set to 'u', does not properly handle characters immediately after a UTF-8 character, which allows remote attackers to conduct cross-site scripting (XSS) attacks via a crafted string.", "affected_packages": [ @@ -29,4 +28,4 @@ "date_published": "2010-04-26T00:00:00+00:00", "weaknesses": [], "url": "https://github.com/rubysec/ruby-advisory-db" -} \ No newline at end of file +} diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index ceedde20f..ecddbfde7 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -1429,3 +1429,71 @@ def test_advisory_content_id_recomputed(self): self.advisory1.unique_content_id, "a15d4651cb05e3513c12263a11e34bd9103f68833cac8f7ffdbbd71b9cb4cf16", ) + + +class TestDropOsvAliasesMigration(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0142_advisoryv2_is_curation_advisoryv2_resolves_todos" + migrate_to = "0143_drop_osv_aliases" + + def setUpBeforeMigration(self, apps): + AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2") + AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias") + Alias = apps.get_model("vulnerabilities", "Alias") + + self.cve_alias = AdvisoryAlias.objects.create(alias="CVE-2010-1330") + self.osvdb_alias = AdvisoryAlias.objects.create(alias="OSV-77297") + self.modern_osv_alias = AdvisoryAlias.objects.create(alias="OSV-2020-001") + + self.legacy_osvdb_alias = Alias.objects.create(alias="OSV-77297") + self.legacy_cve_alias = Alias.objects.create(alias="CVE-2010-1330") + self.legacy_modern_osv = Alias.objects.create(alias="OSV-2020-001") + + self.advisory1 = AdvisoryV2.objects.create( + unique_content_id="old_content_id_hash", + url="https://github.com/rubysec/ruby-advisory-db/blob/master/gems/jruby/CVE-2010-1330.yml", + summary="Test advisory with OSVDB alias", + advisory_id="gems/jruby/CVE-2010-1330", + avid="ruby_importer_v2/gems/jruby/CVE-2010-1330", + datasource_id="ruby_advisory_db", + pipeline_id="ruby_importer_v2", + ) + self.advisory1.aliases.add(self.cve_alias, self.osvdb_alias) + + self.advisory2 = AdvisoryV2.objects.create( + unique_content_id="modern_osv_content_id_hash", + url="https://osv.dev/vulnerability/OSV-2020-001", + summary="Modern OSV advisory", + advisory_id="OSV-2020-001", + avid="github_osv_importer_v2/OSV-2020-001", + datasource_id="github_osv", + pipeline_id="github_osv_importer_v2", + ) + self.advisory2.aliases.add(self.modern_osv_alias) + + def test_osvdb_aliases_dropped(self): + AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias") + Alias = apps.get_model("vulnerabilities", "Alias") + + self.assertFalse(AdvisoryAlias.objects.filter(alias="OSV-77297").exists()) + self.assertFalse(Alias.objects.filter(alias="OSV-77297").exists()) + + def test_modern_osv_and_cve_aliases_preserved(self): + AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias") + Alias = apps.get_model("vulnerabilities", "Alias") + + self.assertTrue(AdvisoryAlias.objects.filter(alias="CVE-2010-1330").exists()) + self.assertTrue(AdvisoryAlias.objects.filter(alias="OSV-2020-001").exists()) + + self.assertTrue(Alias.objects.filter(alias="CVE-2010-1330").exists()) + self.assertTrue(Alias.objects.filter(alias="OSV-2020-001").exists()) + + def test_advisory_unique_content_id_recomputed(self): + self.advisory1.refresh_from_db() + self.assertNotEqual(self.advisory1.unique_content_id, "old_content_id_hash") + self.assertEqual( + list(self.advisory1.aliases.values_list("alias", flat=True)), ["CVE-2010-1330"] + ) + + self.advisory2.refresh_from_db() + self.assertEqual(self.advisory2.unique_content_id, "modern_osv_content_id_hash")