10 year database for v6 (#137)

* Env variable for 10 year db Signed-off-by: Prabhu Subramanian <[email protected]> * Fix version was missing for certain packages for osv Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · May 28, 2024 · bfa1223 · bfa1223
1 parent 6b065c1
commit bfa1223
Show file tree

Hide file tree

Showing 6 changed files with 150 additions and 15 deletions.
diff --git a/.gitignore b/.gitignore
@@ -136,3 +136,4 @@ dmypy.json
 .pyre/
 .coverage
 reports/
+.mise.toml
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "appthreat-vulnerability-db"
-version = "6.0.2"
+version = "6.0.3"
 description = "AppThreat's vulnerability database and package search library with a built-in sqlite based storage. OSV, CVE, GitHub, npm are the primary sources of vulnerabilities."
 authors = [
     {name = "Team AppThreat", email = "[email protected]"},

diff --git a/test/data/osv-pypi-ujson-bug.json b/test/data/osv-pypi-ujson-bug.json
@@ -0,0 +1,110 @@
+{
+  "id": "GHSA-wpqr-jcpx-745r",
+  "summary": "Incorrect handling of invalid surrogate pair characters",
+  "details": "### Impact\n_What kind of vulnerability is it? Who is impacted?_\n\nAnyone parsing JSON from an untrusted source is vulnerable.\n\nJSON strings that contain escaped surrogate characters not part of a proper surrogate pair were decoded incorrectly. Besides corrupting strings, this allowed for potential key confusion and value overwriting in dictionaries.\n\nExamples:\n\n```python\n# An unpaired high surrogate character is ignored.\n>>> ujson.loads(r'\"\\uD800\"')\n''\n>>> ujson.loads(r'\"\\uD800hello\"')\n'hello'\n\n# An unpaired low surrogate character is preserved.\n>>> ujson.loads(r'\"\\uDC00\"')\n'\\udc00'\n\n# A pair of surrogates with additional non surrogate characters pair up in spite of being invalid.\n>>> ujson.loads(r'\"\\uD800foo bar\\uDC00\"')\n'foo bar\ud800\udc00'\n```\n\n### Patches\n_Has the problem been patched? What versions should users upgrade to?_\n\nUsers should upgrade to UltraJSON 5.4.0.\n\nFrom version 5.4.0, UltraJSON decodes lone surrogates in the same way as the standard library's `json` module does, preserving them in the parsed output:\n\n```python3\n>>> ujson.loads(r'\"\\uD800\"')\n'\\ud800'\n>>> ujson.loads(r'\"\\uD800hello\"')\n'\\ud800hello'\n>>> ujson.loads(r'\"\\uDC00\"')\n'\\udc00'\n>>> ujson.loads(r'\"\\uD800foo bar\\uDC00\"')\n'\\ud800foo bar\\udc00'\n```\n\n### Workarounds\n_Is there a way for users to fix or remediate the vulnerability without upgrading?_\n\nShort of switching to an entirely different JSON library, there are no safe alternatives to upgrading.\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in [UltraJSON](http://github.com/ultrajson/ultrajson/issues)\n",
+  "aliases": [
+    "CVE-2022-31116"
+  ],
+  "modified": "2024-02-21T05:24:44.762164Z",
+  "published": "2022-07-05T21:06:00Z",
+  "database_specific": {
+    "nvd_published_at": "2022-07-05T18:15:00Z",
+    "cwe_ids": [
+      "CWE-670"
+    ],
+    "severity": "HIGH",
+    "github_reviewed": true,
+    "github_reviewed_at": "2022-07-05T21:06:00Z"
+  },
+  "references": [
+    {
+      "type": "WEB",
+      "url": "https://github.com/ultrajson/ultrajson/security/advisories/GHSA-wpqr-jcpx-745r"
+    },
+    {
+      "type": "ADVISORY",
+      "url": "https://nvd.nist.gov/vuln/detail/CVE-2022-31116"
+    },
+    {
+      "type": "WEB",
+      "url": "https://github.com/ultrajson/ultrajson/commit/67ec07183342589d602e0fcf7bb1ff3e19272687"
+    },
+    {
+      "type": "PACKAGE",
+      "url": "https://github.com/ultrajson/ultrajson"
+    },
+    {
+      "type": "WEB",
+      "url": "https://lists.fedoraproject.org/archives/list/[email protected]/message/NAU5N4A7EUK2AMUCOLYDD5ARXAJYZBD2"
+    },
+    {
+      "type": "WEB",
+      "url": "https://lists.fedoraproject.org/archives/list/[email protected]/message/OPPU5FZP3LCTXYORFH7NHUMYA5X66IA7"
+    }
+  ],
+  "affected": [
+    {
+      "package": {
+        "name": "ujson",
+        "ecosystem": "PyPI",
+        "purl": "pkg:pypi/ujson"
+      },
+      "ranges": [
+        {
+          "type": "ECOSYSTEM",
+          "events": [
+            {
+              "introduced": "0"
+            },
+            {
+              "fixed": "5.4.0"
+            }
+          ]
+        }
+      ],
+      "versions": [
+        "1.15",
+        "1.18",
+        "1.19",
+        "1.21",
+        "1.22",
+        "1.23",
+        "1.30",
+        "1.33",
+        "1.34",
+        "1.35",
+        "1.4",
+        "1.6",
+        "1.8",
+        "1.9",
+        "2.0.0",
+        "2.0.1",
+        "2.0.2",
+        "2.0.3",
+        "3.0.0",
+        "3.1.0",
+        "3.2.0",
+        "4.0.0",
+        "4.0.1",
+        "4.0.2",
+        "4.1.0",
+        "4.2.0",
+        "4.3.0",
+        "5.0.0",
+        "5.1.0",
+        "5.2.0",
+        "5.3.0"
+      ],
+      "database_specific": {
+        "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2022/07/GHSA-wpqr-jcpx-745r/GHSA-wpqr-jcpx-745r.json"
+      }
+    }
+  ],
+  "schema_version": "1.6.0",
+  "severity": [
+    {
+      "type": "CVSS_V3",
+      "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+    }
+  ]
+}
diff --git a/test/test_source.py b/test/test_source.py
@@ -135,6 +135,15 @@ def test_osv_pypi2_json():
         return json.loads(fp.read())
 
 
+@pytest.fixture
+def test_osv_pypi3_json():
+    test_cve_data = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "data", "osv-pypi-ujson-bug.json"
+    )
+    with open(test_cve_data, "r") as fp:
+        return json.loads(fp.read())
+
+
 @pytest.fixture
 def test_osv_swift_json():
     test_cve_data = os.path.join(
@@ -663,6 +672,7 @@ def test_osv_convert(
     test_osv_go_json,
     test_osv_pypi_json,
     test_osv_pypi2_json,
+    test_osv_pypi3_json,
     test_osv_swift_json,
     test_osv_swift2_json,
     test_osv_mevents_json,
@@ -809,6 +819,9 @@ def test_osv_convert(
     # pypi2
     cve_data = osvlatest.convert(test_osv_pypi2_json)
     assert not cve_data
+    # pypi3
+    cve_data = osvlatest.convert(test_osv_pypi3_json)
+    assert cve_data
 
 
 def test_osv_convert2(test_osv_npm_star_json):

diff --git a/vdb/lib/config.py b/vdb/lib/config.py
@@ -71,13 +71,15 @@
 
 # These feeds introduce too much false positives
 if os.getenv("OSV_INCLUDE_FUZZ"):
-    osv_url_dict[
-        "linux"
-    ] = "https://osv-vulnerabilities.storage.googleapis.com/Linux/all.zip"
-    osv_url_dict[
-        "oss-fuzz"
-    ] = "https://osv-vulnerabilities.storage.googleapis.com/OSS-Fuzz/all.zip"
-    osv_url_dict["android"] = "https://osv-vulnerabilities.storage.googleapis.com/Android/all.zip",
+    osv_url_dict["linux"] = (
+        "https://osv-vulnerabilities.storage.googleapis.com/Linux/all.zip"
+    )
+    osv_url_dict["oss-fuzz"] = (
+        "https://osv-vulnerabilities.storage.googleapis.com/OSS-Fuzz/all.zip"
+    )
+    osv_url_dict["android"] = (
+        "https://osv-vulnerabilities.storage.googleapis.com/Android/all.zip",
+    )
 
 VULN_LIST_URL = "https://github.com/appthreat/vuln-list/archive/refs/heads/main.zip"
 
@@ -146,4 +148,13 @@
 )
 
 # URL for the pre-compiled database
-VDB_DATABASE_URL = os.getenv("VDB_DATABASE_URL", "ghcr.io/appthreat/vdbxz:v6")
+# ghcr.io/appthreat/vdbxz:v6 - Start year 2018
+# ghcr.io/appthreat/vdbxz-10y:v6 - Start year 2018
+VDB_DATABASE_URL = os.getenv(
+    "VDB_DATABASE_URL",
+    (
+        "ghcr.io/appthreat/vdbxz-10y:v6"
+        if os.getenv("USE_VDB_10Y", "") in ("true", "1")
+        else "ghcr.io/appthreat/vdbxz:v6"
+    ),
+)
diff --git a/vdb/lib/osv.py b/vdb/lib/osv.py
@@ -264,15 +264,15 @@ def to_vuln(cve_data):
             # Problem 2: The versions_list may be unsorted and trying to sort based on semantic versions can fail
             # Solution: We do our best to sort the versions_list. If it fails, we assume the input is sorted and store the first and last entry as min and max.
             # The assumption seems to be working as of today, but could result in false positives or false negatives in the future.
-            needs_version_backup = True
+            needs_version_backup = False
+            possible_fix_version = ""
             for r in ranges:
                 events = r.get("events")
                 for ev in events:
                     if ev.get("introduced", "") in (0, "0", "0.0.0"):
-                        break
-                    if ev.get("fixed") or ev.get("last_affected"):
-                        needs_version_backup = False
-                        break
+                        needs_version_backup = True
+                    if ev.get("fixed"):
+                        possible_fix_version = ev.get("fixed")
             if needs_version_backup and len(versions_list) > 1:
                 try:
                     min_ver = min(versions_list, key=Version.parse)
@@ -296,7 +296,7 @@ def to_vuln(cve_data):
                         version_end_including=max_ver,
                         version_start_excluding="",
                         version_end_excluding="",
-                        fix_version_start_including="",
+                        fix_version_start_including=possible_fix_version,
                         fix_version_end_including="",
                         fix_version_start_excluding=max_ver,
                         fix_version_end_excluding="",