Merge branch 'master' into renovate/functions

google · Aug 7, 2024 · f87c031 · f87c031
2 parents 4154a9c + baa360d
commit f87c031
Show file tree

Hide file tree

Showing 23 changed files with 7,214 additions and 1,356 deletions.
diff --git a/.github/workflows/staleness.yml b/.github/workflows/staleness.yml
@@ -12,13 +12,21 @@ jobs:
           days-before-stale: 60
           days-before-close: 14
           operations-per-run: 100
-          stale-issue-label: stale
-          stale-pr-label: stale
           remove-stale-when-updated: true
           exempt-issue-labels: "good first issue,backlog"
           exempt-all-assignees: true
           ignore-updates: false
-          stale-issue-message: "This issue has not had any activity for 60 days and will be automatically closed in two weeks"
-          stale-pr-message: "This pull request has not had any activity for 60 days and will be automatically closed in two weeks"
-          close-issue-message: "Automatically closing stale issue"
-          close-pr-message: "Automatically closing stale pull request"
+          stale-issue-label: stale
+          stale-issue-message: |
+            This issue has not had any activity for 60 days and will be automatically closed in two weeks
+
+            See https://github.com/google/osv.dev/blob/master/CONTRIBUTING.md for how to contribute a PR if you're interested in helping out.
+          stale-pr-label: stale
+          stale-pr-message: |
+            This pull request has not had any activity for 60 days and will be automatically closed in two weeks
+          close-issue-label: "autoclosed"
+          close-issue-message: |
+            Automatically closing stale issue
+          close-pr-label: "autoclosed"
+          close-pr-message: |
+            Automatically closing stale pull request
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/docker/importer/importer_test.py b/docker/importer/importer_test.py
@@ -76,7 +76,7 @@ def setUp(self):
         type=osv.SourceRepositoryType.GIT,
         id='oss-fuzz',
         name='oss-fuzz',
-        db_prefix='OSV-',
+        db_prefix=['OSV-'],
         repo_url='file://' + self.remote_source_repo_path,
         repo_username='',
         ignore_patterns=['.*IGNORE.*'])
@@ -381,15 +381,17 @@ def test_no_updates(self, mock_publish):  # pylint: disable=unused-argument
   @mock.patch('google.cloud.pubsub_v1.PublisherClient.publish')
   def test_ignore(self, mock_publish):  # pylint: disable=unused-argument
     """Test ignoring."""
-    # TODO(michaelkedar): This test doesn't check anything
-    self.skipTest("Not Implemented")
-    self.mock_repo.add_file('2021-111IGNORE.yaml', _MIN_VALID_VULNERABILITY)
-    self.mock_repo.commit('User', 'user@email', 'message.')
-
-    imp = importer.Importer('fake_public_key', 'fake_private_key', self.tmp_dir,
-                            importer.DEFAULT_PUBLIC_LOGGING_BUCKET, 'bucket',
-                            True, False)
-    imp.run()
+    self.assertTrue(self.source_repo.ignore_file('/tmp/foo/recoredIGNOREme'))
+    source_repo_ignore_negative = osv.SourceRepository(
+        ignore_patterns=['(^(?!USN-).*$)'])
+    self.assertTrue(
+        source_repo_ignore_negative.ignore_file('/tmp/foo/CVE-2024-1234.json'))
+    source_repo_ignore_multiple = osv.SourceRepository(
+        ignore_patterns=['^(?!MAL-).*$', 'MAL-0000.*'])
+    self.assertTrue(
+        source_repo_ignore_multiple.ignore_file('/tmp/foo/CVE-2024-1234.json'))
+    self.assertTrue(
+        source_repo_ignore_multiple.ignore_file('/tmp/foo/MAL-0000-0001.json'))
 
 
 @mock.patch('importer.utcnow', lambda: datetime.datetime(2021, 1, 1))
@@ -700,7 +702,7 @@ def setUp(self):
         bucket='osv-test-cve-osv-conversion',
         directory_path='osv-output',
         extension='.json',
-        db_prefix='CVE-')
+        db_prefix=['CVE-'])
 
     tests.mock_datetime(self)
 
@@ -763,7 +765,7 @@ def setUp(self):
         name='curl',
         link=MOCK_ADDRESS_FORMAT,
         rest_api_url=MOCK_ADDRESS_FORMAT,
-        db_prefix='CURL-',
+        db_prefix=['CURL-'],
         extension='.json',
         editable=False)
     self.source_repo.put()

diff --git a/docker/worker/testdata/UpdateTest_dont_index_too_many_git_versions.txt b/docker/worker/testdata/UpdateTest_dont_index_too_many_git_versions.txt
diff --git a/docker/worker/worker_test.py b/docker/worker/worker_test.py
@@ -148,7 +148,8 @@ def setUp(self):
     tests.mock_clone(self, return_value=pygit2.Repository('osv-test'))
     tests.mock_datetime(self)
 
-    osv.SourceRepository(id='oss-fuzz', name='oss-fuzz', db_prefix='OSV-').put()
+    osv.SourceRepository(
+        id='oss-fuzz', name='oss-fuzz', db_prefix=['OSV-']).put()
 
     allocated_bug = osv.Bug(
         db_id='OSV-2020-1337',
@@ -516,7 +517,8 @@ def setUp(self):
 
   def test_mark_bug_invalid(self):
     """Test mark_bug_invalid."""
-    osv.SourceRepository(id='oss-fuzz', name='oss-fuzz', db_prefix='OSV-').put()
+    osv.SourceRepository(
+        id='oss-fuzz', name='oss-fuzz', db_prefix=['OSV-']).put()
     osv.Bug(db_id='OSV-2021-1', source_id='oss-fuzz:1337').put()
     osv.AffectedCommits(bug_id='OSV-2021-1').put()
     osv.AffectedCommits(bug_id='OSV-2021-1').put()
@@ -734,7 +736,7 @@ def setUp(self):
         type=osv.SourceRepositoryType.GIT,
         id='source',
         name='source',
-        db_prefix='BLAH-',
+        db_prefix=['OSV-'],
         repo_url='file://' + self.remote_source_repo_path,
         editable=True,
         repo_username='',
@@ -1124,6 +1126,7 @@ def test_update_pypi(self):
     self.source_repo.ignore_git = False
     self.source_repo.versions_from_repo = False
     self.source_repo.detect_cherrypicks = False
+    self.source_repo.db_prefix.append('PYSEC-')
     self.source_repo.put()
 
     self.mock_repo.add_file(
@@ -1151,9 +1154,8 @@ def test_update_pypi(self):
     diff = repo.diff(commit.parents[0], commit)
     self.expect_equal('diff_pypi', diff.patch)
 
-    self.expect_dict_equal(
-        'update_pypi',
-        ndb.Key(osv.Bug, 'source:PYSEC-123').get()._to_dict())
+    self.expect_dict_equal('update_pypi',
+                           ndb.Key(osv.Bug, 'PYSEC-123').get()._to_dict())
 
     affected_commits = list(osv.AffectedCommits.query())
     self.assertEqual(1, len(affected_commits))
@@ -1174,6 +1176,7 @@ def test_normalize_pypi(self):
     self.source_repo.ignore_git = False
     self.source_repo.versions_from_repo = False
     self.source_repo.detect_cherrypicks = False
+    self.source_repo.db_prefix.append('PYSEC-')
     self.source_repo.put()
 
     self.mock_repo.add_file(
@@ -1200,7 +1203,7 @@ def test_normalize_pypi(self):
 
     self.expect_dict_equal(
         'normalized_pypi',
-        ndb.Key(osv.Bug, 'source:PYSEC-456').get()._to_dict(),
+        ndb.Key(osv.Bug, 'PYSEC-456').get()._to_dict(),
     )
 
     affected_commits = list(osv.AffectedCommits.query())
@@ -1223,6 +1226,7 @@ def test_update_last_affected(self):
     self.source_repo.ignore_git = False
     self.source_repo.versions_from_repo = False
     self.source_repo.detect_cherrypicks = False
+    self.source_repo.db_prefix.append('PYSEC-')
     self.source_repo.put()
 
     self.mock_repo.add_file(
@@ -1252,14 +1256,15 @@ def test_update_last_affected(self):
 
     self.expect_dict_equal(
         'update_last_affected',
-        ndb.Key(osv.Bug, 'source:PYSEC-124').get()._to_dict(),
+        ndb.Key(osv.Bug, 'PYSEC-124').get()._to_dict(),
     )
 
   def test_update_maven(self):
     """Test updating maven."""
     self.source_repo.ignore_git = False
     self.source_repo.versions_from_repo = False
     self.source_repo.detect_cherrypicks = False
+    self.source_repo.db_prefix.append('GHSA-')
     self.source_repo.put()
 
     self.mock_repo.add_file(
@@ -1290,7 +1295,7 @@ def test_update_maven(self):
 
     self.expect_dict_equal(
         'update_maven',
-        ndb.Key(osv.Bug, 'source:GHSA-838r-hvwh-24h8').get()._to_dict(),
+        ndb.Key(osv.Bug, 'GHSA-838r-hvwh-24h8').get()._to_dict(),
     )
 
     self.mock_publish.assert_not_called()
@@ -1300,6 +1305,7 @@ def test_update_linux(self):
     self.source_repo.ignore_git = False
     self.source_repo.versions_from_repo = False
     self.source_repo.detect_cherrypicks = False
+    self.source_repo.db_prefix.append('LINUX-')
     self.source_repo.put()
 
     self.mock_repo.add_file(
@@ -1320,7 +1326,7 @@ def test_update_linux(self):
 
     self.expect_dict_equal(
         'update_linux',
-        ndb.Key(osv.Bug, 'source:LINUX-123').get()._to_dict(),
+        ndb.Key(osv.Bug, 'LINUX-123').get()._to_dict(),
     )
 
     affected_commits = list(osv.AffectedCommits.query())
@@ -1340,6 +1346,7 @@ def test_update_bucket(self):
     self.source_repo.type = osv.SourceRepositoryType.BUCKET
     self.source_repo.bucket = TEST_BUCKET
     self.source_repo.editable = False
+    self.source_repo.db_prefix.append('GO-')
     self.source_repo.put()
 
     task_runner = worker.TaskRunner(ndb_client, None, self.tmp_dir.name, None,
@@ -1363,6 +1370,7 @@ def test_update_debian(self):
     self.source_repo.ignore_git = False
     self.source_repo.versions_from_repo = False
     self.source_repo.detect_cherrypicks = False
+    self.source_repo.db_prefix.append('DSA-')
     self.source_repo.put()
 
     self.mock_repo.add_file(
@@ -1393,7 +1401,7 @@ def test_update_debian(self):
 
     self.expect_dict_equal(
         'update_debian',
-        ndb.Key(osv.Bug, 'source:DSA-3029-1').get()._to_dict(),
+        ndb.Key(osv.Bug, 'DSA-3029-1').get()._to_dict(),
     )
 
     self.mock_publish.assert_not_called()
@@ -1403,6 +1411,7 @@ def test_update_alpine(self):
     self.source_repo.ignore_git = False
     self.source_repo.versions_from_repo = False
     self.source_repo.detect_cherrypicks = False
+    self.source_repo.db_prefix.append('CVE-')
     self.source_repo.put()
 
     self.mock_repo.add_file(
@@ -1434,7 +1443,7 @@ def test_update_alpine(self):
 
     self.expect_dict_equal(
         'update_alpine',
-        ndb.Key(osv.Bug, 'source:CVE-2022-27449').get()._to_dict(),
+        ndb.Key(osv.Bug, 'CVE-2022-27449').get()._to_dict(),
     )
 
     self.mock_publish.assert_not_called()
@@ -1444,6 +1453,7 @@ def test_update_android(self):
     self.source_repo.type = osv.SourceRepositoryType.BUCKET
     self.source_repo.bucket = TEST_BUCKET
     self.source_repo.editable = False
+    self.source_repo.db_prefix.append('ASB-A-')
     self.source_repo.put()
 
     task_runner = worker.TaskRunner(ndb_client, None, self.tmp_dir.name, None,
@@ -1544,6 +1554,7 @@ def test_update_bucket_cve(self):
     self.source_repo.type = osv.SourceRepositoryType.BUCKET
     self.source_repo.bucket = TEST_BUCKET
     self.source_repo.editable = False
+    self.source_repo.db_prefix.append('CVE-')
     self.source_repo.put()
 
     task_runner = worker.TaskRunner(ndb_client, None, self.tmp_dir.name, None,
@@ -1604,7 +1615,7 @@ def test_last_affected_git(self):
 
     self.expect_dict_equal(
         'last_affected_git',
-        ndb.Key(osv.Bug, 'source:OSV-TEST-last-affected-01').get()._to_dict(),
+        ndb.Key(osv.Bug, 'OSV-TEST-last-affected-01').get()._to_dict(),
     )
 
     affected_commits = list(osv.AffectedCommits.query())
@@ -1620,6 +1631,51 @@ def test_last_affected_git(self):
         [codecs.encode(commit, 'hex') for commit in affected_commits.commits],
     )
 
+  def test_invalid_prefix(self):
+    """Test attempting to create a bug with a invalid db_prefix."""
+    with self.assertRaises(ValueError):
+      # Default db_prefix is `OSV-`
+      osv.Bug(
+          db_id='BLAH-131',
+          project=['blah.com/package'],
+          ecosystem=['ecosystem'],
+          source_id='source:OSV-131.yaml',
+          import_last_modified=datetime.datetime(2021, 1, 1, 0, 0),
+          source_of_truth=osv.SourceOfTruth.SOURCE_REPO,
+      ).put()
+
+  def test_dont_index_too_many_git_versions(self):
+    """Test that we don't index too many versions from Git."""
+    self.source_repo.ignore_git = False
+    self.source_repo.versions_from_repo = True
+    self.source_repo.detect_cherrypicks = True
+    self.source_repo.put()
+
+    # Use any valid OSV input test file here.
+    self.mock_repo.add_file(
+        'OSV-TEST-last-affected-01.yaml',
+        self._load_test_data(
+            os.path.join(TEST_DATA_DIR, 'OSV-TEST-last-affected-01.yaml')),
+    )
+    self.mock_repo.commit('User', 'user@email')
+    task_runner = worker.TaskRunner(ndb_client, None, self.tmp_dir.name, None,
+                                    None)
+    message = mock.Mock()
+    message.attributes = {
+        'source': 'source',
+        'path': 'OSV-TEST-last-affected-01.yaml',
+        'original_sha256': _sha256('OSV-TEST-last-affected-01.yaml'),
+        'deleted': 'false',
+    }
+    task_runner._source_update(message)
+
+    bug = ndb.Key(osv.Bug, 'OSV-TEST-last-affected-01').get()
+
+    # Manually append versions over the expected version limit.
+    bug.affected_packages[0].versions = ['%05d' % i for i in range(5001)]
+    bug.put()
+    self.expect_dict_equal('dont_index_too_many_git_versions', bug._to_dict())
+
 
 if __name__ == '__main__':
   ds_emulator = tests.start_datastore_emulator()