Drop legacy OSV aliases from Ruby advisories

vulnerabilities/importers/ruby.py

@@ -145,8 +145,6 @@ def get_aliases(record) -> [str]:
     aliases = []
     if record.get("cve"):
         aliases.append("CVE-{}".format(record.get("cve")))
-    if record.get("osvdb"):
-        aliases.append("OSV-{}".format(record.get("osvdb")))
     if record.get("ghsa"):
         aliases.append("GHSA-{}".format(record.get("ghsa")))
     return aliases

vulnerabilities/migrations/0143_drop_osv_aliases.py

@@ -0,0 +1,156 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from django.db import migrations
+
+from vulnerabilities.importer import AdvisoryDataV2
+from vulnerabilities.importer import AffectedPackageV2
+from vulnerabilities.importer import PatchData
+from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.utils import compute_content_id_v2
+from vulnerabilities.utils import normalize_list
+from vulnerabilities.utils import purl_to_dict
+
+"""
+Drop legacy OSVDB-derived aliases imported from rubygem data sources.
+
+The rubysec advisory data contains legacy references to the defunct OSVDB
+database in its `osvdb` field, and these were imported as `OSV-<number>`
+aliases. These are not public aliases and collide with the modern OSV.dev
+namespace. Genuine OSV.dev identifiers have the form `OSV-<year>-<number>`
+and are preserved.
+
+See https://github.com/aboutcode-org/vulnerablecode/issues/2421
+"""
+
+OSVDB_DERIVED_ALIAS_REGEX = r"^OSV-\d+$"
+
+
+def commit_patch_to_dict(patch):
+    return {
+        "vcs_url": patch.vcs_url,
+        "commit_hash": patch.commit_hash,
+        "patch_text": patch.patch_text,
+        "patch_checksum": patch.patch_checksum,
+    }
+
+
+def to_affected_package_data(impact):
+    """Return `AffectedPackageV2` data from the impact."""
+    return AffectedPackageV2.from_dict(
+        {
+            "package": purl_to_dict(impact.base_purl),
+            "affected_version_range": impact.affecting_vers,
+            "fixed_version_range": impact.fixed_vers,
+            "introduced_by_commit_patches": [
+                commit_patch_to_dict(commit)
+                for commit in impact.introduced_by_package_commit_patches.all()
+            ],
+            "fixed_by_commit_patches": [
+                commit_patch_to_dict(commit)
+                for commit in impact.fixed_by_package_commit_patches.all()
+            ],
+        }
+    )
+
+
+def to_patch_data(patch):
+    """Return `PatchData` from the Patch."""
+    return PatchData.from_dict(
+        {
+            "patch_url": patch.patch_url,
+            "patch_text": patch.patch_text,
+            "patch_checksum": patch.patch_checksum,
+        }
+    )
+
+
+def to_reference_v2_data(ref):
+    return ReferenceV2.from_dict(
+        {
+            "reference_id": ref.reference_id,
+            "reference_type": ref.reference_type,
+            "url": ref.url,
+        }
+    )
+
+
+def to_vulnerability_severity_data(severity):
+    return VulnerabilitySeverity.from_dict(
+        {
+            "system": severity.scoring_system,
+            "value": severity.value,
+            "scoring_elements": severity.scoring_elements,
+            "published_at": severity.published_at,
+            "url": severity.url,
+        }
+    )
+
+
+def to_advisory_data(advisory):
+    return AdvisoryDataV2(
+        advisory_id=advisory.advisory_id,
+        aliases=normalize_list([item.alias for item in advisory.aliases.all()]),
+        summary=advisory.summary,
+        affected_packages=normalize_list(
+            [to_affected_package_data(impacted) for impacted in advisory.impacted_packages.all()]
+        ),
+        references=normalize_list([to_reference_v2_data(ref) for ref in advisory.references.all()]),
+        patches=normalize_list([to_patch_data(patch) for patch in advisory.patches.all()]),
+        date_published=advisory.date_published,
+        weaknesses=normalize_list([weak.cwe_id for weak in advisory.weaknesses.all()]),
+        severities=normalize_list(
+            [to_vulnerability_severity_data(sev) for sev in advisory.severities.all()]
+        ),
+        url=advisory.url,
+    )
+
+
+def drop_osv_aliases(apps, schema_editor):
+    AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias")
+    AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2")
+    Alias = apps.get_model("vulnerabilities", "Alias")
+
+    # Identify affected advisories before deleting associations
+    advisory_ids = list(
+        AdvisoryV2.objects.filter(aliases__alias__iregex=OSVDB_DERIVED_ALIAS_REGEX)
+        .distinct()
+        .values_list("id", flat=True)
+    )
+
+    # Delete legacy OSV- (OSVDB-derived) aliases
+    AdvisoryAlias.objects.filter(alias__iregex=OSVDB_DERIVED_ALIAS_REGEX).delete()
+    Alias.objects.filter(alias__iregex=OSVDB_DERIVED_ALIAS_REGEX).delete()
+
+    # Recompute unique_content_id for affected advisories
+    batch = []
+    batch_size = 2000
+    for advisory in AdvisoryV2.objects.filter(id__in=advisory_ids).iterator(chunk_size=1000):
+        advisory.unique_content_id = compute_content_id_v2(to_advisory_data(advisory))
+        batch.append(advisory)
+        if len(batch) >= batch_size:
+            AdvisoryV2.objects.bulk_update(batch, ["unique_content_id"])
+            batch.clear()
+    if batch:
+        AdvisoryV2.objects.bulk_update(batch, ["unique_content_id"])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0142_advisoryv2_is_curation_advisoryv2_resolves_todos"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            drop_osv_aliases,
+            reverse_code=migrations.RunPython.noop,
+        ),
+    ]

vulnerabilities/pipelines/v2_importers/ruby_importer.py

@@ -202,8 +202,6 @@ def get_aliases(record) -> [str]:
     aliases = []
     if record.get("cve"):
         aliases.append("CVE-{}".format(record.get("cve")))
-    if record.get("osvdb"):
-        aliases.append("OSV-{}".format(record.get("osvdb")))
     if record.get("ghsa"):
         aliases.append("GHSA-{}".format(record.get("ghsa")))
     return aliases

vulnerabilities/tests/pipelines/v2_importers/test_ruby_importer_v2.py

@@ -37,3 +37,39 @@ def test_ruby_advisories_per_file(yml_file):
 
     expected_file = yml_file.with_name(yml_file.stem + "-expected.json")
     util_tests.check_results_against_json(result, expected_file)
+
+
+from vulnerabilities.pipelines.v2_importers.ruby_importer import get_aliases
+
+
+def test_get_aliases_drops_osvdb_alias():
+    record = {
+        "cve": "2020-0001",
+        "ghsa": "xxxx-yyyy-zzzz",
+        "osvdb": 12345,
+    }
+    aliases = get_aliases(record)
+    assert aliases == ["CVE-2020-0001", "GHSA-xxxx-yyyy-zzzz"]
+    assert "OSV-12345" not in aliases
+
+
+def test_get_aliases_only_osvdb():
+    record = {
+        "osvdb": 65123,
+    }
+    assert get_aliases(record) == []
+
+
+def test_get_aliases_no_osvdb():
+    record = {
+        "cve": "2021-1234",
+    }
+    assert get_aliases(record) == ["CVE-2021-1234"]
+
+
+def test_get_aliases_osvdb_and_ghsa_only():
+    record = {
+        "ghsa": "xxxx-yyyy-zzzz",
+        "osvdb": 12345,
+    }
+    assert get_aliases(record) == ["GHSA-xxxx-yyyy-zzzz"]

vulnerabilities/tests/test_data/ruby/CVE-2010-1330-expected.json

@@ -1,7 +1,6 @@
 {
   "aliases": [
-    "CVE-2010-1330",
-    "OSV-77297"
+    "CVE-2010-1330"
   ],
   "summary": "CVE-2010-1330 jruby: XSS in the regular expression engine when processing invalid UTF-8 byte sequences\nThe regular expression engine in JRuby before 1.4.1, when $KCODE is set to 'u', does not properly handle characters immediately after a UTF-8 character, which allows remote attackers to conduct cross-site scripting (XSS) attacks via a crafted string.",
   "affected_packages": [
@@ -29,4 +28,4 @@
   "date_published": "2010-04-26T00:00:00+00:00",
   "weaknesses": [],
   "url": "https://github.com/rubysec/ruby-advisory-db"
-}
+}

vulnerabilities/tests/test_data_migrations.py

@@ -1429,3 +1429,71 @@ def test_advisory_content_id_recomputed(self):
             self.advisory1.unique_content_id,
             "a15d4651cb05e3513c12263a11e34bd9103f68833cac8f7ffdbbd71b9cb4cf16",
         )
+
+
+class TestDropOsvAliasesMigration(TestMigrations):
+    app_name = "vulnerabilities"
+    migrate_from = "0142_advisoryv2_is_curation_advisoryv2_resolves_todos"
+    migrate_to = "0143_drop_osv_aliases"
+
+    def setUpBeforeMigration(self, apps):
+        AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2")
+        AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias")
+        Alias = apps.get_model("vulnerabilities", "Alias")
+
+        self.cve_alias = AdvisoryAlias.objects.create(alias="CVE-2010-1330")
+        self.osvdb_alias = AdvisoryAlias.objects.create(alias="OSV-77297")
+        self.modern_osv_alias = AdvisoryAlias.objects.create(alias="OSV-2020-001")
+
+        self.legacy_osvdb_alias = Alias.objects.create(alias="OSV-77297")
+        self.legacy_cve_alias = Alias.objects.create(alias="CVE-2010-1330")
+        self.legacy_modern_osv = Alias.objects.create(alias="OSV-2020-001")
+
+        self.advisory1 = AdvisoryV2.objects.create(
+            unique_content_id="old_content_id_hash",
+            url="https://github.com/rubysec/ruby-advisory-db/blob/master/gems/jruby/CVE-2010-1330.yml",
+            summary="Test advisory with OSVDB alias",
+            advisory_id="gems/jruby/CVE-2010-1330",
+            avid="ruby_importer_v2/gems/jruby/CVE-2010-1330",
+            datasource_id="ruby_advisory_db",
+            pipeline_id="ruby_importer_v2",
+        )
+        self.advisory1.aliases.add(self.cve_alias, self.osvdb_alias)
+
+        self.advisory2 = AdvisoryV2.objects.create(
+            unique_content_id="modern_osv_content_id_hash",
+            url="https://osv.dev/vulnerability/OSV-2020-001",
+            summary="Modern OSV advisory",
+            advisory_id="OSV-2020-001",
+            avid="github_osv_importer_v2/OSV-2020-001",
+            datasource_id="github_osv",
+            pipeline_id="github_osv_importer_v2",
+        )
+        self.advisory2.aliases.add(self.modern_osv_alias)
+
+    def test_osvdb_aliases_dropped(self):
+        AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias")
+        Alias = apps.get_model("vulnerabilities", "Alias")
+
+        self.assertFalse(AdvisoryAlias.objects.filter(alias="OSV-77297").exists())
+        self.assertFalse(Alias.objects.filter(alias="OSV-77297").exists())
+
+    def test_modern_osv_and_cve_aliases_preserved(self):
+        AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias")
+        Alias = apps.get_model("vulnerabilities", "Alias")
+
+        self.assertTrue(AdvisoryAlias.objects.filter(alias="CVE-2010-1330").exists())
+        self.assertTrue(AdvisoryAlias.objects.filter(alias="OSV-2020-001").exists())
+
+        self.assertTrue(Alias.objects.filter(alias="CVE-2010-1330").exists())
+        self.assertTrue(Alias.objects.filter(alias="OSV-2020-001").exists())
+
+    def test_advisory_unique_content_id_recomputed(self):
+        self.advisory1.refresh_from_db()
+        self.assertNotEqual(self.advisory1.unique_content_id, "old_content_id_hash")
+        self.assertEqual(
+            list(self.advisory1.aliases.values_list("alias", flat=True)), ["CVE-2010-1330"]
+        )
+
+        self.advisory2.refresh_from_db()
+        self.assertEqual(self.advisory2.unique_content_id, "modern_osv_content_id_hash")