From 26a9749ac15ef4e4c5f6c33cb921464156097ad2 Mon Sep 17 00:00:00 2001 From: Prabhu Subramanian Date: Thu, 17 Nov 2022 11:59:37 +0000 Subject: [PATCH] OS packages from OSV --- .github/workflows/pythonapp.yml | 4 +- .github/workflows/pythonpublish.yml | 6 +- setup.py | 13 ++++- test/test_utils.py | 91 ++++++++++++++++++++++++++++- vdb/lib/__init__.py | 10 ++++ vdb/lib/config.py | 8 +-- vdb/lib/osv.py | 20 +++++-- vdb/lib/utils.py | 84 ++++++++++++++++++++++++++ 8 files changed, 219 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 437eb6f..9766445 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -5,10 +5,10 @@ on: [push] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: - os: [ubuntu-20.04, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] python-version: ['3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index b7dcabf..fac2780 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -9,14 +9,14 @@ on: jobs: deploy: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: '3.10' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/setup.py b/setup.py index f2d83db..567bb4a 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="appthreat-vulnerability-db", - version="2.1.0", + version="3.0.0", author="Team AppThreat", author_email="cloud@appthreat.com", description="AppThreat's vulnerability database and package search library with a built-in file based storage. OSV, CVE, GitHub, npm are the primary sources of vulnerabilities.", @@ -14,7 +14,16 @@ long_description_content_type="text/markdown", url="https://github.com/appthreat/vulnerability-db", packages=setuptools.find_packages(), - install_requires=["requests", "appdirs", "tabulate", "msgpack", "orjson", "semver", "packageurl-python", "cvss"], + install_requires=[ + "requests", + "appdirs", + "tabulate", + "msgpack", + "orjson", + "semver", + "packageurl-python", + "cvss", + ], classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", diff --git a/test/test_utils.py b/test/test_utils.py index 165e179..db76da1 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -107,6 +107,20 @@ def test_version_compare(): assert res res = utils.version_compare("2.0.27.RELEASE", "*", "2.0.27", None, None) assert res + res = utils.version_compare("0.6.18", "0", "0.6.19-r0", None, None) + assert res + res = utils.version_compare("2.15.4-r0", "0", "2.14.1-r0", None, None) + assert not res + res = utils.version_compare("1.8.19", "0", "1.8.20_p2-r0", None, None) + assert res + res = utils.version_compare("5.8.9", "0", "6.0_p20170701-r0", None, None) + assert res + res = utils.version_compare("1.5", "0", "1.6_rc2-r5", None, None) + assert res + res = utils.version_compare("0.99.1", "0", "0.99.4-r0", None, None) + assert res + res = utils.version_compare("0.7.0", "0", "0.7.1_alpha-r0", None, None) + assert res def test_version_compare1(): @@ -207,10 +221,83 @@ def test_version_build_diff_compare(): def test_version_hash_compare(): - res = utils.version_compare("3.1.2", "0", None, None, "acb672b6a179567632e032f547582f30fa2f4aa7") + res = utils.version_compare( + "3.1.2", "0", None, None, "acb672b6a179567632e032f547582f30fa2f4aa7" + ) + assert not res + res = utils.version_compare( + "3.1.2", + "acb672b6a179567632e032f547582f30fa2f4aa7", + "acb672b6a179567632e032f547582f30fa2f4aa7", + None, + None, + ) assert not res - res = utils.version_compare("3.1.2", "acb672b6a179567632e032f547582f30fa2f4aa7", "acb672b6a179567632e032f547582f30fa2f4aa7", None, None) + res = utils.version_compare( + "8b626d45127d6f5ada7d815b83cfdc09e8cb1394", + "8b626d45127d6f5ada7d815b83cfdc09e8cb1394", + "8b626d45127d6f5ada7d815b83cfdc09e8cb1394", + None, + None, + ) + assert res + + +def test_os_build_compare(): + res = utils.version_compare("2.8.4-1+squeeze4", "0", "2.8.6-1+squeeze4") + assert res + res = utils.version_compare("2.8.4-7+squeeze4", "0", "2.8.6-1+squeeze4") + assert res + res = utils.version_compare("2.6.34-1squeeze8", "0", "2.6.32-48squeeze8") assert not res + res = utils.version_compare("3.0.1-8+wheezy6", "0", "3.0.4-3+wheezy6") + assert res + res = utils.version_compare("1:51.2.1-1~deb7u1", "0", "1:52.2.1-1~deb7u1") + assert res + res = utils.version_compare("2:51.2.1-1~deb7u1", "0", "1:52.2.1-1~deb7u1") + assert res + res = utils.version_compare("6:51.2.1-1~deb7u1", "0", "2:52.2.1-1~deb7u1") + assert res + res = utils.version_compare( + "1.2.0-1.2+wheezy4+deb7u1", "0", "1.2.1-2.2+wheezy4+deb7u1" + ) + assert res + res = utils.version_compare("8:7.2.947-7+deb7u4", "0", "2:7.3.547-7+deb7u4") + assert res + res = utils.version_compare("9.04~dfsg-6.3+deb7u7", "0", "9.05~dfsg-6.3+deb7u7") + assert res + res = utils.version_compare("1:1.7.5.4-1+wheezy5", "0", "1:1.7.10.4-1+wheezy5") + assert res + res = utils.version_compare("1:1.7.11.1-1+wheezy5", "0", "1:1.7.10.4-1+wheezy5") + assert not res + res = utils.version_compare( + "7u180-2.6.14-2~deb8u1", "7u179-2.6.14-2~deb8u1", "7u181-2.6.14-2~deb8u1" + ) + assert res + res = utils.version_compare( + "7u182-2.8.14-2~deb8u1", "7u179-2.6.14-2~deb8u1", "7u181-2.6.14-2~deb8u1" + ) + assert res + res = utils.version_compare( + "1.0.8~git20140621.1.440916e+dfsg1-13~deb8u3", + "0", + "1.1.0~git20140921.1.440916e+dfsg1-13~deb8u3", + ) + assert res + res = utils.version_compare("2:3.25-1+debu8u4", "0", "2:3.26-1+debu8u4") + assert res + res = utils.version_compare( + "2.4.9", "0", "2.5.0.26054~ReleaseCandidate3.ds2-1+squeeze1" + ) + assert res + res = utils.version_compare("2.4.15-7.woody.2.1", "0", "2.4.17-2.woody.2.2") + assert res + res = utils.version_compare( + "0.12.0+git20120207.aaa852f-1+deb7u1", + "0", + "0.12.1+git20120407.aaa852f-1+deb7u1", + ) + assert res def test_parse_uri(): diff --git a/vdb/lib/__init__.py b/vdb/lib/__init__.py index e4bf77a..4407a1e 100644 --- a/vdb/lib/__init__.py +++ b/vdb/lib/__init__.py @@ -25,6 +25,12 @@ "hex", "packagist", "uvi", + "apk", + "deb", + "rpm", + "linux", + "swid", + "oss-fuzz", ] # Maps variations of string to package types @@ -40,6 +46,10 @@ "crates": ["rust", "crates.io", "cargo"], "pub": ["dart"], "hex": ["elixir"], + "github": ["actions"], + "apk": ["alpine"], + "deb": ["ubuntu", "debian"], + "rpm": ["redhat", "centos"], } # CPE Regex diff --git a/vdb/lib/config.py b/vdb/lib/config.py index 136dc4a..325eb86 100644 --- a/vdb/lib/config.py +++ b/vdb/lib/config.py @@ -52,14 +52,14 @@ "pub": "https://osv-vulnerabilities.storage.googleapis.com/Pub/all.zip", "uvi": "https://osv-vulnerabilities.storage.googleapis.com/UVI/all.zip", "github": "https://osv-vulnerabilities.storage.googleapis.com/GitHub%20Actions/all.zip", -} - -# Unused -osv_os_url_dict = { "android": "https://osv-vulnerabilities.storage.googleapis.com/Android/all.zip", + "alpine": "https://osv-vulnerabilities.storage.googleapis.com/Alpine/all.zip", + "gsd": "https://osv-vulnerabilities.storage.googleapis.com/GSD/all.zip", "linux": "https://osv-vulnerabilities.storage.googleapis.com/Linux/all.zip", "debian": "https://osv-vulnerabilities.storage.googleapis.com/Debian/all.zip", + "oss-fuzz": "https://osv-vulnerabilities.storage.googleapis.com/OSS-Fuzz/all.zip", } + # CVE types to exclude - hardware nvd_exclude_types = ["h"] if os.getenv("NVD_EXCLUDE_TYPES") is not None: diff --git a/vdb/lib/osv.py b/vdb/lib/osv.py index 2046118..64da60d 100644 --- a/vdb/lib/osv.py +++ b/vdb/lib/osv.py @@ -99,7 +99,9 @@ def to_vuln(self, cve_data): {} {} """.format( - cve_data.get("summary"), cve_data.get("details"), aliases_block + cve_data.get("summary", "Summary"), + cve_data.get("details", ""), + aliases_block, ) references = [] # Change the key from type to name in references @@ -108,7 +110,8 @@ def to_vuln(self, cve_data): references = json_lib.dumps(references) if isinstance(references, bytes): references = references.decode("utf-8", "ignore") - if not cve_id.startswith("RUSTSEC"): + # Try to locate the CVE id from the aliases section + if not cve_id.startswith("CVE") and not cve_id.startswith("RUSTSEC"): for i in aliases: if not i.startswith("OSV"): cve_id = i @@ -130,8 +133,13 @@ def to_vuln(self, cve_data): # Issue 58 cve_database_specific = cve_data.get("database_specific") cve_ecosystem_specific = cve_data.get("ecosystem_specific") - if cve_database_specific and cve_database_specific.get("severity"): - severity = cve_database_specific.get("severity") + if cve_database_specific: + if cve_database_specific.get("severity"): + severity = cve_database_specific.get("severity") + if cve_database_specific.get("cwe_ids"): + cwes = cve_database_specific.get("cwe_ids") + if isinstance(cwes, list): + cwe_id = ",".join(cwes) if cve_ecosystem_specific and cve_ecosystem_specific.get("severity"): severity = cve_ecosystem_specific.get("severity") for pkg_data in cve_data.get("affected"): @@ -183,6 +191,10 @@ def to_vuln(self, cve_data): if len(tmpA) == 2: vendor = tmpA[0] pkg_name = tmpA[-1] + # For OS packages, such as alpine OSV appends the os version to the vendor which is weird + # Let's remove it to keep things sane + if ":" in vendor: + vendor = vendor.split(":")[0] for r in ranges: events = r.get("events") versions_list = r.get("versions", []) diff --git a/vdb/lib/utils.py b/vdb/lib/utils.py index ebcae17..849f2a0 100644 --- a/vdb/lib/utils.py +++ b/vdb/lib/utils.py @@ -36,6 +36,12 @@ KNOWN_PRERELEASE_STR = ["final", "release", "alpha", "beta", "rc", "latest"] +DEBIAN_VALID_VERSION = re.compile( + r"^((?P\d+):)?" + "(?P[A-Za-z0-9.+:~-]+?)" + "(-(?P[A-Za-z0-9+.~]+))?$" +) + class ClassNotFoundError(Exception): """docstring for ClassNotFoundError""" @@ -327,6 +333,15 @@ def checkHex(s): return all(c in hex_digits for c in s) +def checkEpoch(s): + if not s: + return False + m = DEBIAN_VALID_VERSION.search(s) + if not m: + return False + return True if m.group("epoch") is not None else False + + def is_hash_mode( compare_ver, min_version, @@ -343,6 +358,53 @@ def is_hash_mode( ) +def is_epoch_mode( + compare_ver, + min_version, + max_version, + min_affected_version_excluding, + max_affected_version_excluding, +): + return ( + checkEpoch(compare_ver) + or checkEpoch(min_version) + or checkEpoch(max_version) + or checkEpoch(min_affected_version_excluding) + or checkEpoch(max_affected_version_excluding) + ) + + +def trim_epoch( + compare_ver, + min_version, + max_version, + min_affected_version_excluding, + max_affected_version_excluding, +): + if checkEpoch(compare_ver): + m = DEBIAN_VALID_VERSION.search(compare_ver) + compare_ver = m.group("upstream_version") + if checkEpoch(min_version): + m = DEBIAN_VALID_VERSION.search(min_version) + min_version = m.group("upstream_version") + if checkEpoch(max_version): + m = DEBIAN_VALID_VERSION.search(max_version) + max_version = m.group("upstream_version") + if checkEpoch(min_affected_version_excluding): + m = DEBIAN_VALID_VERSION.search(min_affected_version_excluding) + min_affected_version_excluding = m.group("upstream_version") + if checkEpoch(max_affected_version_excluding): + m = DEBIAN_VALID_VERSION.search(max_affected_version_excluding) + max_affected_version_excluding = m.group("upstream_version") + return ( + compare_ver, + min_version, + max_version, + min_affected_version_excluding, + max_affected_version_excluding, + ) + + def version_compare( compare_ver, min_version, @@ -368,6 +430,28 @@ def version_compare( min_affected_version_excluding, max_affected_version_excluding, ) + # Debian OS packages could have epoch. Detect and extract the upstream version + epoch_mode_detected = is_epoch_mode( + compare_ver, + min_version, + max_version, + min_affected_version_excluding, + max_affected_version_excluding, + ) + if epoch_mode_detected: + ( + compare_ver, + min_version, + max_version, + min_affected_version_excluding, + max_affected_version_excluding, + ) = trim_epoch( + compare_ver, + min_version, + max_version, + min_affected_version_excluding, + max_affected_version_excluding, + ) # Semver compatible and including versions provided is_min_exclude = False is_max_exclude = False