Merge branch 'master' into renovate/lock-file-maintenance

.github/workflows/staleness.yml

@@ -12,13 +12,21 @@ jobs:
           days-before-stale: 60
           days-before-close: 14
           operations-per-run: 100
-          stale-issue-label: stale
-          stale-pr-label: stale
           remove-stale-when-updated: true
           exempt-issue-labels: "good first issue,backlog"
           exempt-all-assignees: true
           ignore-updates: false
-          stale-issue-message: "This issue has not had any activity for 60 days and will be automatically closed in two weeks"
-          stale-pr-message: "This pull request has not had any activity for 60 days and will be automatically closed in two weeks"
-          close-issue-message: "Automatically closing stale issue"
-          close-pr-message: "Automatically closing stale pull request"
+          stale-issue-label: stale
+          stale-issue-message: |
+            This issue has not had any activity for 60 days and will be automatically closed in two weeks
+
+            See https://github.com/google/osv.dev/blob/master/CONTRIBUTING.md for how to contribute a PR if you're interested in helping out.
+          stale-pr-label: stale
+          stale-pr-message: |
+            This pull request has not had any activity for 60 days and will be automatically closed in two weeks
+          close-issue-label: "autoclosed"
+          close-issue-message: |
+            Automatically closing stale issue
+          close-pr-label: "autoclosed"
+          close-pr-message: |
+            Automatically closing stale pull request

docker/worker/worker_test.py

@@ -1644,6 +1644,38 @@ def test_invalid_prefix(self):
           source_of_truth=osv.SourceOfTruth.SOURCE_REPO,
       ).put()
 
+  def test_dont_index_too_many_git_versions(self):
+    """Test that we don't index too many versions from Git."""
+    self.source_repo.ignore_git = False
+    self.source_repo.versions_from_repo = True
+    self.source_repo.detect_cherrypicks = True
+    self.source_repo.put()
+
+    # Use any valid OSV input test file here.
+    self.mock_repo.add_file(
+        'OSV-TEST-last-affected-01.yaml',
+        self._load_test_data(
+            os.path.join(TEST_DATA_DIR, 'OSV-TEST-last-affected-01.yaml')),
+    )
+    self.mock_repo.commit('User', 'user@email')
+    task_runner = worker.TaskRunner(ndb_client, None, self.tmp_dir.name, None,
+                                    None)
+    message = mock.Mock()
+    message.attributes = {
+        'source': 'source',
+        'path': 'OSV-TEST-last-affected-01.yaml',
+        'original_sha256': _sha256('OSV-TEST-last-affected-01.yaml'),
+        'deleted': 'false',
+    }
+    task_runner._source_update(message)
+
+    bug = ndb.Key(osv.Bug, 'OSV-TEST-last-affected-01').get()
+
+    # Manually append versions over the expected version limit.
+    bug.affected_packages[0].versions = ['%05d' % i for i in range(5001)]
+    bug.put()
+    self.expect_dict_equal('dont_index_too_many_git_versions', bug._to_dict())
+
 
 if __name__ == '__main__':
   ds_emulator = tests.start_datastore_emulator()

docs/Gemfile.lock

@@ -236,7 +236,7 @@ GEM
     rb-fsevent (0.11.2)
     rb-inotify (0.11.1)
       ffi (~> 1.0)
-    rexml (3.3.2)
+    rexml (3.3.3)
       strscan
     rouge (3.26.0)
     rubyzip (2.3.2)

gcp/api/Pipfile

@@ -5,8 +5,8 @@ name = "pypi"
 
 [packages]
 google-cloud-ndb = "==2.3.1"
-google-cloud-logging = "==3.10.0"
-packageurl-python = "==0.15.1"
+google-cloud-logging = "==3.11.0"
+packageurl-python = "==0.15.6"
 packaging = "==20.9"
 requests = "==2.32.3"
 grpcio = "==1.64.1"

gcp/api/pyproject.toml

@@ -6,8 +6,8 @@ package-mode = false
 python = "^3.11"
 
 google-cloud-ndb = "==2.3.1"
-google-cloud-logging = "==3.10.0"
-packageurl-python = "==0.15.1"
+google-cloud-logging = "==3.11.0"
+packageurl-python = "==0.15.6"
 packaging = "==20.9"
 requests = "==2.32.3"
 grpcio = "==1.64.1"

gcp/api/server.py

@@ -1065,17 +1065,9 @@ def query_by_version(context: QueryContext,
       # Query for non-enumerated ecosystems.
       bugs, next_page_token = yield _query_by_comparing_versions(
           context, query, ecosystem, version)
-      logging.info(
-          '[_query_by_comparing_versions] Package %s '
-          'at version %s has total %d bugs in %s', package_name or purl,
-          version, len(bugs), ecosystem)
     else:
       bugs, next_page_token = yield _query_by_generic_version(
           context, query, package_name, ecosystem, purl, version)
-      logging.info(
-          '[_query_by_generic_version] Package %s '
-          'at version %s has total %d bugs in %s', package_name or purl,
-          version, len(bugs), ecosystem)
 
   else:
     logging.warning("Package query without ecosystem specified")
@@ -1267,7 +1259,11 @@ def _is_affected(ecosystem: str, version: str,
     if affected:
       return True
 
-  return False
+  # OSV allows users to add affected versions
+  # that are not covered by affected ranges.
+  # TODO(gongh@): Move this check before the version range check
+  # after performance analysis.
+  return version in affected_package.versions
 
 
 def main():

osv/models.py

@@ -15,6 +15,7 @@
 
 import datetime
 import enum
+import logging
 import re
 import os
 
@@ -35,6 +36,8 @@
 
 SCHEMA_VERSION = '1.6.0'
 
+_MAX_GIT_VERSIONS_TO_INDEX = 5000
+
 
 def _check_valid_severity(prop, value):
   """Check valid severity."""
@@ -429,11 +432,25 @@ def _pre_put_hook(self):  # pylint: disable=arguments-differ
         # No need to normalize if the ecosystem is supported.
         self.affected_fuzzy.extend(affected_package.versions)
       else:
-        self.affected_fuzzy.extend(
-            bug.normalize_tags(
-                _maybe_strip_repo_prefixes(
-                    affected_package.versions,
-                    [range.repo_url for range in affected_package.ranges])))
+        if (not affected_package.package.ecosystem and
+            len(affected_package.versions) > _MAX_GIT_VERSIONS_TO_INDEX):
+          # Assume that if there is no ecosystem specified, then these versions
+          # were enumerated from Git.
+          #
+          # Mitigate cases where the Git repo tag matching results in too many
+          # versions to index for Datastore.
+          # It's OK to do this because the primary intended matching mechanism
+          # for Git is via commit hash matching instead.
+          logging.info(
+              'Skipping indexing of git versions for %s '
+              'as there are too many (%s).', self.db_id,
+              len(affected_package.versions))
+        else:
+          self.affected_fuzzy.extend(
+              bug.normalize_tags(
+                  _maybe_strip_repo_prefixes(
+                      affected_package.versions,
+                      [range.repo_url for range in affected_package.ranges])))
 
       self.has_affected |= bool(affected_package.versions)
 

source.yaml

@@ -6,7 +6,7 @@
   repo_url: https://github.com/AlmaLinux/osv-database.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: ALBA-
+  db_prefix: ['ALBA-']
   ignore_git: False
   human_link: 'https://errata.almalinux.org/{{ ECOSYSTEMS[1].split(":")[1] }}/{{ BUG_ID | replace(":", "-", 1) }}.html'
   link: https://github.com/AlmaLinux/osv-database/blob/master/
@@ -20,7 +20,7 @@
   repo_url: https://github.com/AlmaLinux/osv-database.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: ALEA-
+  db_prefix: ['ALEA-']
   ignore_git: False
   human_link: 'https://errata.almalinux.org/{{ ECOSYSTEMS[1].split(":")[1] }}/{{ BUG_ID | replace(":", "-", 1) }}.html'
   link: https://github.com/AlmaLinux/osv-database/blob/master/
@@ -34,7 +34,7 @@
   repo_url: https://github.com/AlmaLinux/osv-database.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: ALSA-
+  db_prefix: ['ALSA-']
   ignore_git: False
   human_link: 'https://errata.almalinux.org/{{ ECOSYSTEMS[1].split(":")[1] }}/{{ BUG_ID | replace(":", "-", 1) }}.html'
   link: https://github.com/AlmaLinux/osv-database/blob/master/
@@ -47,7 +47,7 @@
   detect_cherrypicks: False
   extension: .json
   bucket: android-osv
-  db_prefix: A-
+  db_prefix: ['A-', 'ASB-A', 'PUB-A']
   ignore_git: True
   link: https://storage.googleapis.com/android-osv/
   editable: False
@@ -60,7 +60,7 @@
   repo_url: https://github.com/bitnami/vulndb.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: BIT-
+  db_prefix: ['BIT-']
   ignore_git: False
   link: https://github.com/bitnami/vulndb/tree/main/
   editable: False
@@ -73,7 +73,7 @@
   directory_path: 'chainguard/osv'
   detect_cherrypicks: False
   extension: '.json'
-  db_prefix: 'CGA-'
+  db_prefix: ['CGA-']
   ignore_git: True
   link: 'https://packages.cgr.dev/chainguard/osv/'
   editable: False
@@ -86,7 +86,7 @@
   directory_path: docs
   detect_cherrypicks: False
   extension: .json
-  db_prefix: CURL-
+  db_prefix: ['CURL-']
   ignore_git: True
   human_link: 'https://curl.se/docs/{{ BUG_ID | replace("CURL-", "") }}.html'
   link: https://curl.se/docs/
@@ -100,7 +100,7 @@
   detect_cherrypicks: False
   extension: .json
   bucket: cve-osv-conversion
-  db_prefix: CVE-
+  db_prefix: ['CVE-']
   ignore_git: False
   human_link: 'https://nvd.nist.gov/vuln/detail/{{ BUG_ID }}'
   link: https://storage.googleapis.com/cve-osv-conversion/
@@ -114,7 +114,7 @@
   detect_cherrypicks: False
   extension: .json
   bucket: debian-osv
-  db_prefix: DLA-
+  db_prefix: ['DLA-']
   ignore_git: True
   human_link: 'https://security-tracker.debian.org/tracker/{{ BUG_ID }}'
   link: https://storage.googleapis.com/debian-osv/
@@ -128,7 +128,7 @@
   detect_cherrypicks: False
   extension: .json
   bucket: debian-osv
-  db_prefix: DSA-
+  db_prefix: ['DSA-']
   ignore_git: True
   human_link: 'https://security-tracker.debian.org/tracker/{{ BUG_ID }}'
   link: https://storage.googleapis.com/debian-osv/
@@ -142,7 +142,7 @@
   detect_cherrypicks: False
   extension: .json
   bucket: debian-osv
-  db_prefix: DTSA-
+  db_prefix: ['DTSA-']
   ignore_git: True
   human_link: 'https://security-tracker.debian.org/tracker/{{ BUG_ID }}'
   link: https://storage.googleapis.com/debian-osv/
@@ -156,7 +156,7 @@
   repo_url: https://github.com/github/advisory-database.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: GHSA-
+  db_prefix: ['GHSA-']
   ignore_git: True
   human_link: 'https://github.com/advisories/{{ BUG_ID }}'
   link: https://github.com/github/advisory-database/blob/main/
@@ -170,7 +170,7 @@
   detect_cherrypicks: True
   extension: .json
   bucket: go-vulndb
-  db_prefix: GO-
+  db_prefix: ['GO-']
   ignore_git: True
   human_link: 'https://pkg.go.dev/vuln/{{ BUG_ID }}'
   link: https://vuln.go.dev/
@@ -184,7 +184,7 @@
   repo_url: https://github.com/haskell/security-advisories.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: HSEC-
+  db_prefix: ['HSEC-']
   ignore_git: False
   link: https://github.com/haskell/security-advisories/blob/generated/osv-export/
   editable: False
@@ -198,7 +198,7 @@
   repo_url: https://github.com/ossf/malicious-packages.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: MAL-
+  db_prefix: ['MAL-']
   ignore_git: False
   link: https://github.com/ossf/malicious-packages/blob/main/
   editable: False
@@ -211,7 +211,7 @@
   repo_url: ssh://github.com/google/oss-fuzz-vulns
   detect_cherrypicks: True
   extension: .yaml
-  db_prefix: OSV-
+  db_prefix: ['OSV-']
   ignore_git: False
   link: https://github.com/google/oss-fuzz-vulns/blob/main/
   editable: True
@@ -225,7 +225,7 @@
   repo_url: https://github.com/psf/advisory-database.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: PSF-
+  db_prefix: ['PSF-']
   ignore_git: False
   link: https://github.com/psf/advisory-database/blob/main/
   editable: False
@@ -238,7 +238,7 @@
   repo_url: ssh://github.com/pypa/advisory-database
   detect_cherrypicks: False
   extension: .yaml
-  db_prefix: PYSEC-
+  db_prefix: ['PYSEC-']
   ignore_git: False
   link: https://github.com/pypa/advisory-database/blob/main/
   editable: False
@@ -252,7 +252,7 @@
   repo_url: https://github.com/RConsortium/r-advisory-database.git
   detect_cherrypicks: False
   extension: .yaml
-  db_prefix: RSEC-
+  db_prefix: ['RSEC-']
   ignore_git: False
   link: https://github.com/RConsortium/r-advisory-database/blob/main/
   editable: False
@@ -264,7 +264,7 @@
   detect_cherrypicks: False
   extension: .json
   bucket: resf-osv-data
-  db_prefix: RLSA-
+  db_prefix: ['RLSA-']
   ignore_git: False
   link: https://storage.googleapis.com/resf-osv-data/
   editable: False
@@ -276,7 +276,7 @@
   detect_cherrypicks: False
   extension: .json
   bucket: resf-osv-data
-  db_prefix: RXSA-
+  db_prefix: ['RXSA-']
   ignore_git: False
   link: https://storage.googleapis.com/resf-osv-data/
   editable: False
@@ -290,7 +290,7 @@
   repo_url: https://github.com/rustsec/advisory-db.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: RUSTSEC-
+  db_prefix: ['RUSTSEC-']
   ignore_git: False
   human_link: 'https://rustsec.org/advisories/{{ BUG_ID }}'
   link: https://github.com/rustsec/advisory-db/blob/osv/
@@ -305,7 +305,7 @@
   repo_url: 'https://github.com/canonical/ubuntu-security-notices.git'
   detect_cherrypicks: False
   extension: '.json'
-  db_prefix: 'USN-'
+  db_prefix: ['USN-']
   ignore_git: False
   human_link: 'https://ubuntu.com/security/notices/{{ BUG_ID }}'
   link: 'https://github.com/canonical/ubuntu-security-notices/blob/main/'
@@ -318,7 +318,7 @@
   repo_url: https://github.com/cloudsecurityalliance/gsd-database.git
   detect_cherrypicks: False
   extension: .json
-  db_prefix: GSD-
+  db_prefix: ['GSD-']
   ignore_git: False
   link: https://github.com/cloudsecurityalliance/gsd-database/blob/main/
   editable: False