OS packages from OSV

AppThreat · Nov 17, 2022 · 26a9749 · 26a9749
1 parent ebe5be2
commit 26a9749
Show file tree

Hide file tree

Showing 8 changed files with 219 additions and 17 deletions.
diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml
@@ -5,10 +5,10 @@ on: [push]
 jobs:
   build:
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ['3.8', '3.9', '3.10']
     steps:
     - uses: actions/checkout@v3

diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml
@@ -9,14 +9,14 @@ on:
 jobs:
   deploy:
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.x'
+          python-version: '3.10'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="appthreat-vulnerability-db",
-    version="2.1.0",
+    version="3.0.0",
     author="Team AppThreat",
     author_email="[email protected]",
     description="AppThreat's vulnerability database and package search library with a built-in file based storage. OSV, CVE, GitHub, npm are the primary sources of vulnerabilities.",
@@ -14,7 +14,16 @@
     long_description_content_type="text/markdown",
     url="https://github.com/appthreat/vulnerability-db",
     packages=setuptools.find_packages(),
-    install_requires=["requests", "appdirs", "tabulate", "msgpack", "orjson", "semver", "packageurl-python", "cvss"],
+    install_requires=[
+        "requests",
+        "appdirs",
+        "tabulate",
+        "msgpack",
+        "orjson",
+        "semver",
+        "packageurl-python",
+        "cvss",
+    ],
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",

diff --git a/test/test_utils.py b/test/test_utils.py
@@ -107,6 +107,20 @@ def test_version_compare():
     assert res
     res = utils.version_compare("2.0.27.RELEASE", "*", "2.0.27", None, None)
     assert res
+    res = utils.version_compare("0.6.18", "0", "0.6.19-r0", None, None)
+    assert res
+    res = utils.version_compare("2.15.4-r0", "0", "2.14.1-r0", None, None)
+    assert not res
+    res = utils.version_compare("1.8.19", "0", "1.8.20_p2-r0", None, None)
+    assert res
+    res = utils.version_compare("5.8.9", "0", "6.0_p20170701-r0", None, None)
+    assert res
+    res = utils.version_compare("1.5", "0", "1.6_rc2-r5", None, None)
+    assert res
+    res = utils.version_compare("0.99.1", "0", "0.99.4-r0", None, None)
+    assert res
+    res = utils.version_compare("0.7.0", "0", "0.7.1_alpha-r0", None, None)
+    assert res
 
 
 def test_version_compare1():
@@ -207,10 +221,83 @@ def test_version_build_diff_compare():
 
 
 def test_version_hash_compare():
-    res = utils.version_compare("3.1.2", "0", None, None, "acb672b6a179567632e032f547582f30fa2f4aa7")
+    res = utils.version_compare(
+        "3.1.2", "0", None, None, "acb672b6a179567632e032f547582f30fa2f4aa7"
+    )
+    assert not res
+    res = utils.version_compare(
+        "3.1.2",
+        "acb672b6a179567632e032f547582f30fa2f4aa7",
+        "acb672b6a179567632e032f547582f30fa2f4aa7",
+        None,
+        None,
+    )
     assert not res
-    res = utils.version_compare("3.1.2", "acb672b6a179567632e032f547582f30fa2f4aa7", "acb672b6a179567632e032f547582f30fa2f4aa7", None, None)
+    res = utils.version_compare(
+        "8b626d45127d6f5ada7d815b83cfdc09e8cb1394",
+        "8b626d45127d6f5ada7d815b83cfdc09e8cb1394",
+        "8b626d45127d6f5ada7d815b83cfdc09e8cb1394",
+        None,
+        None,
+    )
+    assert res
+
+
+def test_os_build_compare():
+    res = utils.version_compare("2.8.4-1+squeeze4", "0", "2.8.6-1+squeeze4")
+    assert res
+    res = utils.version_compare("2.8.4-7+squeeze4", "0", "2.8.6-1+squeeze4")
+    assert res
+    res = utils.version_compare("2.6.34-1squeeze8", "0", "2.6.32-48squeeze8")
     assert not res
+    res = utils.version_compare("3.0.1-8+wheezy6", "0", "3.0.4-3+wheezy6")
+    assert res
+    res = utils.version_compare("1:51.2.1-1~deb7u1", "0", "1:52.2.1-1~deb7u1")
+    assert res
+    res = utils.version_compare("2:51.2.1-1~deb7u1", "0", "1:52.2.1-1~deb7u1")
+    assert res
+    res = utils.version_compare("6:51.2.1-1~deb7u1", "0", "2:52.2.1-1~deb7u1")
+    assert res
+    res = utils.version_compare(
+        "1.2.0-1.2+wheezy4+deb7u1", "0", "1.2.1-2.2+wheezy4+deb7u1"
+    )
+    assert res
+    res = utils.version_compare("8:7.2.947-7+deb7u4", "0", "2:7.3.547-7+deb7u4")
+    assert res
+    res = utils.version_compare("9.04~dfsg-6.3+deb7u7", "0", "9.05~dfsg-6.3+deb7u7")
+    assert res
+    res = utils.version_compare("1:1.7.5.4-1+wheezy5", "0", "1:1.7.10.4-1+wheezy5")
+    assert res
+    res = utils.version_compare("1:1.7.11.1-1+wheezy5", "0", "1:1.7.10.4-1+wheezy5")
+    assert not res
+    res = utils.version_compare(
+        "7u180-2.6.14-2~deb8u1", "7u179-2.6.14-2~deb8u1", "7u181-2.6.14-2~deb8u1"
+    )
+    assert res
+    res = utils.version_compare(
+        "7u182-2.8.14-2~deb8u1", "7u179-2.6.14-2~deb8u1", "7u181-2.6.14-2~deb8u1"
+    )
+    assert res
+    res = utils.version_compare(
+        "1.0.8~git20140621.1.440916e+dfsg1-13~deb8u3",
+        "0",
+        "1.1.0~git20140921.1.440916e+dfsg1-13~deb8u3",
+    )
+    assert res
+    res = utils.version_compare("2:3.25-1+debu8u4", "0", "2:3.26-1+debu8u4")
+    assert res
+    res = utils.version_compare(
+        "2.4.9", "0", "2.5.0.26054~ReleaseCandidate3.ds2-1+squeeze1"
+    )
+    assert res
+    res = utils.version_compare("2.4.15-7.woody.2.1", "0", "2.4.17-2.woody.2.2")
+    assert res
+    res = utils.version_compare(
+        "0.12.0+git20120207.aaa852f-1+deb7u1",
+        "0",
+        "0.12.1+git20120407.aaa852f-1+deb7u1",
+    )
+    assert res
 
 
 def test_parse_uri():

diff --git a/vdb/lib/__init__.py b/vdb/lib/__init__.py
@@ -25,6 +25,12 @@
     "hex",
     "packagist",
     "uvi",
+    "apk",
+    "deb",
+    "rpm",
+    "linux",
+    "swid",
+    "oss-fuzz",
 ]
 
 # Maps variations of string to package types
@@ -40,6 +46,10 @@
     "crates": ["rust", "crates.io", "cargo"],
     "pub": ["dart"],
     "hex": ["elixir"],
+    "github": ["actions"],
+    "apk": ["alpine"],
+    "deb": ["ubuntu", "debian"],
+    "rpm": ["redhat", "centos"],
 }
 
 # CPE Regex

diff --git a/vdb/lib/config.py b/vdb/lib/config.py
@@ -52,14 +52,14 @@
     "pub": "https://osv-vulnerabilities.storage.googleapis.com/Pub/all.zip",
     "uvi": "https://osv-vulnerabilities.storage.googleapis.com/UVI/all.zip",
     "github": "https://osv-vulnerabilities.storage.googleapis.com/GitHub%20Actions/all.zip",
-}
-
-# Unused
-osv_os_url_dict = {
     "android": "https://osv-vulnerabilities.storage.googleapis.com/Android/all.zip",
+    "alpine": "https://osv-vulnerabilities.storage.googleapis.com/Alpine/all.zip",
+    "gsd": "https://osv-vulnerabilities.storage.googleapis.com/GSD/all.zip",
     "linux": "https://osv-vulnerabilities.storage.googleapis.com/Linux/all.zip",
     "debian": "https://osv-vulnerabilities.storage.googleapis.com/Debian/all.zip",
+    "oss-fuzz": "https://osv-vulnerabilities.storage.googleapis.com/OSS-Fuzz/all.zip",
 }
+
 # CVE types to exclude - hardware
 nvd_exclude_types = ["h"]
 if os.getenv("NVD_EXCLUDE_TYPES") is not None:

diff --git a/vdb/lib/osv.py b/vdb/lib/osv.py
@@ -99,7 +99,9 @@ def to_vuln(self, cve_data):
 {}
 {}
             """.format(
-            cve_data.get("summary"), cve_data.get("details"), aliases_block
+            cve_data.get("summary", "Summary"),
+            cve_data.get("details", ""),
+            aliases_block,
         )
         references = []
         # Change the key from type to name in references
@@ -108,7 +110,8 @@ def to_vuln(self, cve_data):
         references = json_lib.dumps(references)
         if isinstance(references, bytes):
             references = references.decode("utf-8", "ignore")
-        if not cve_id.startswith("RUSTSEC"):
+        # Try to locate the CVE id from the aliases section
+        if not cve_id.startswith("CVE") and not cve_id.startswith("RUSTSEC"):
             for i in aliases:
                 if not i.startswith("OSV"):
                     cve_id = i
@@ -130,8 +133,13 @@ def to_vuln(self, cve_data):
         # Issue 58
         cve_database_specific = cve_data.get("database_specific")
         cve_ecosystem_specific = cve_data.get("ecosystem_specific")
-        if cve_database_specific and cve_database_specific.get("severity"):
-            severity = cve_database_specific.get("severity")
+        if cve_database_specific:
+            if cve_database_specific.get("severity"):
+                severity = cve_database_specific.get("severity")
+            if cve_database_specific.get("cwe_ids"):
+                cwes = cve_database_specific.get("cwe_ids")
+                if isinstance(cwes, list):
+                    cwe_id = ",".join(cwes)
         if cve_ecosystem_specific and cve_ecosystem_specific.get("severity"):
             severity = cve_ecosystem_specific.get("severity")
         for pkg_data in cve_data.get("affected"):
@@ -183,6 +191,10 @@ def to_vuln(self, cve_data):
                 if len(tmpA) == 2:
                     vendor = tmpA[0]
                     pkg_name = tmpA[-1]
+            # For OS packages, such as alpine OSV appends the os version to the vendor which is weird
+            # Let's remove it to keep things sane
+            if ":" in vendor:
+                vendor = vendor.split(":")[0]
             for r in ranges:
                 events = r.get("events")
                 versions_list = r.get("versions", [])

diff --git a/vdb/lib/utils.py b/vdb/lib/utils.py
@@ -36,6 +36,12 @@
 
 KNOWN_PRERELEASE_STR = ["final", "release", "alpha", "beta", "rc", "latest"]
 
+DEBIAN_VALID_VERSION = re.compile(
+    r"^((?P<epoch>\d+):)?"
+    "(?P<upstream_version>[A-Za-z0-9.+:~-]+?)"
+    "(-(?P<debian_revision>[A-Za-z0-9+.~]+))?$"
+)
+
 
 class ClassNotFoundError(Exception):
     """docstring for ClassNotFoundError"""
@@ -327,6 +333,15 @@ def checkHex(s):
     return all(c in hex_digits for c in s)
 
 
+def checkEpoch(s):
+    if not s:
+        return False
+    m = DEBIAN_VALID_VERSION.search(s)
+    if not m:
+        return False
+    return True if m.group("epoch") is not None else False
+
+
 def is_hash_mode(
     compare_ver,
     min_version,
@@ -343,6 +358,53 @@ def is_hash_mode(
     )
 
 
+def is_epoch_mode(
+    compare_ver,
+    min_version,
+    max_version,
+    min_affected_version_excluding,
+    max_affected_version_excluding,
+):
+    return (
+        checkEpoch(compare_ver)
+        or checkEpoch(min_version)
+        or checkEpoch(max_version)
+        or checkEpoch(min_affected_version_excluding)
+        or checkEpoch(max_affected_version_excluding)
+    )
+
+
+def trim_epoch(
+    compare_ver,
+    min_version,
+    max_version,
+    min_affected_version_excluding,
+    max_affected_version_excluding,
+):
+    if checkEpoch(compare_ver):
+        m = DEBIAN_VALID_VERSION.search(compare_ver)
+        compare_ver = m.group("upstream_version")
+    if checkEpoch(min_version):
+        m = DEBIAN_VALID_VERSION.search(min_version)
+        min_version = m.group("upstream_version")
+    if checkEpoch(max_version):
+        m = DEBIAN_VALID_VERSION.search(max_version)
+        max_version = m.group("upstream_version")
+    if checkEpoch(min_affected_version_excluding):
+        m = DEBIAN_VALID_VERSION.search(min_affected_version_excluding)
+        min_affected_version_excluding = m.group("upstream_version")
+    if checkEpoch(max_affected_version_excluding):
+        m = DEBIAN_VALID_VERSION.search(max_affected_version_excluding)
+        max_affected_version_excluding = m.group("upstream_version")
+    return (
+        compare_ver,
+        min_version,
+        max_version,
+        min_affected_version_excluding,
+        max_affected_version_excluding,
+    )
+
+
 def version_compare(
     compare_ver,
     min_version,
@@ -368,6 +430,28 @@ def version_compare(
         min_affected_version_excluding,
         max_affected_version_excluding,
     )
+    # Debian OS packages could have epoch. Detect and extract the upstream version
+    epoch_mode_detected = is_epoch_mode(
+        compare_ver,
+        min_version,
+        max_version,
+        min_affected_version_excluding,
+        max_affected_version_excluding,
+    )
+    if epoch_mode_detected:
+        (
+            compare_ver,
+            min_version,
+            max_version,
+            min_affected_version_excluding,
+            max_affected_version_excluding,
+        ) = trim_epoch(
+            compare_ver,
+            min_version,
+            max_version,
+            min_affected_version_excluding,
+            max_affected_version_excluding,
+        )
     # Semver compatible and including versions provided
     is_min_exclude = False
     is_max_exclude = False