Merge branch 'master' into renovate/docs

.github/workflows/pr-check.yml

@@ -0,0 +1,35 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: OSV PR format check
+
+on:
+  # `pull_request_target` is only required when editing PRs from forks.
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+
+permissions:
+  pull-requests: read
+
+jobs:
+  title-check:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

deployment/clouddeploy/gke-workers/base/nvd-mirror.yaml

@@ -8,7 +8,7 @@ spec:
   failedJobsHistoryLimit: 3
   jobTemplate:
     spec:
-      activeDeadlineSeconds: 2100
+      activeDeadlineSeconds: 7200
       template:
         spec:
           tolerations:
@@ -33,7 +33,7 @@ spec:
                 memory: "192G"
           nodeSelector:
             cloud.google.com/gke-nodepool: highend
-          restartPolicy: OnFailure
+          restartPolicy: Never
           volumes:
             - name: "ssd"
               hostPath:

docker/indexer/Dockerfile

@@ -24,7 +24,7 @@ RUN go mod download
 COPY ./ /build
 RUN CGO_ENABLED=0 ./build.sh
 
-FROM gcr.io/distroless/base@sha256:5c395fa86ad4eb0613bc9cae7412003f575a25ff9b5cb4845e37e175d2a50bba
+FROM gcr.io/distroless/base-debian12@sha256:1aae189e3baecbb4044c648d356ddb75025b2ba8d14cdc9c2a19ba784c90bfb9
 COPY --from=GO_BUILD build/indexer /indexer
 ENTRYPOINT ["/indexer"]
 CMD ["--help"]

docker/terraform/Dockerfile

@@ -6,7 +6,7 @@ ARG TERRAFORM_VERSION
 WORKDIR /build/
 RUN GOBIN=$(pwd) go install github.com/hashicorp/terraform@v${TERRAFORM_VERSION}
 
-FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:be4424609f5840e86dea3be27ff747af3d2c80b9bd956337085c508bb174e95a
+FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:2166d24be9a61a36adda37476f094f4b14b8527bcf6d997f0555cb9d42ff544b
 
 COPY --from=GO_BUILD /build/terraform /usr/bin/terraform
 COPY entrypoint.bash /builder/entrypoint.bash

docs/Gemfile.lock

@@ -240,7 +240,7 @@ GEM
     rb-fsevent (0.11.2)
     rb-inotify (0.11.1)
       ffi (~> 1.0)
-    rexml (3.3.5)
+    rexml (3.3.6)
       strscan
     rouge (3.30.0)
     rubyzip (2.3.2)

gcp/api/cursor.py

@@ -0,0 +1,147 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""OSV API server cursor implementation."""
+
+import base64
+from enum import Enum
+from typing import Self
+import typing
+
+from google.cloud import ndb
+import google.cloud.ndb.exceptions as ndb_exceptions
+
+_FIRST_PAGE_TOKEN = base64.urlsafe_b64encode(b'FIRST_PAGE_TOKEN').decode()
+# Use ':' as the separator as it doesn't appear in urlsafe_b64encode
+# (which is what is used for both _FIRST_PAGE_TOKEN, and ndb.Cursor.urlsafe())
+_METADATA_SEPARATOR = ':'
+
+
+class _QueryCursorState(Enum):
+  """
+  Stores the 3 states a query cursor can be in.
+  """
+
+  # The cursor has reached the end, no need to return to the user
+  ENDED = 0
+  # The cursor is at the very start of the query, no cursor needs to be
+  # set when querying ndb
+  STARTED = 1
+  # ndb.Cursor is set and in progress
+  IN_PROGRESS = 2
+
+
+class QueryCursor:
+  """
+  Custom cursor class that wraps the ndb cursor.
+  Allows us to represent the "starting" cursor.
+
+  The default state is ENDED with no ndb.Cursor.
+
+  This type could have 3 states encoded in _QueryCursorState.
+  If the current state is IN_PROGRESS, self.cursor will not be None.
+
+  Attributes:
+    query_number: This cursor is specifically for the Nth ndb datastore
+      query in the current query request. (Starts from 1)
+    ndb_cursor: Get the internal ndb_cursor. This could be None.
+    ended: Whether this cursor is for a query that has finished returning data.
+  """
+
+  _ndb_cursor: ndb.Cursor | None = None
+  _cursor_state: _QueryCursorState = _QueryCursorState.ENDED
+  # The first query is numbered 1. This is because the query counter is
+  # incremented **before** the query and the query number being used.
+  query_number: int = 1
+
+  @classmethod
+  def from_page_token(cls, page_token: str | None) -> Self:
+    """Generate a query cursor from a url safe page token."""
+    qc = cls()
+
+    if not page_token:
+      qc._cursor_state = _QueryCursorState.STARTED
+      qc._ndb_cursor = None
+      return qc
+
+    split_values = page_token.split(_METADATA_SEPARATOR, 1)
+    if len(split_values) == 2:
+      page_token = split_values[1]
+      try:
+        qc.query_number = int(split_values[0])
+      except ValueError as e:
+        raise ValueError('Invalid page token.') from e
+
+    if not page_token or page_token == _FIRST_PAGE_TOKEN:
+      qc._cursor_state = _QueryCursorState.STARTED
+      qc._ndb_cursor = None
+      return qc
+
+    qc._ndb_cursor = ndb.Cursor(urlsafe=page_token)
+    qc._cursor_state = _QueryCursorState.IN_PROGRESS
+    return qc
+
+  def update_from_iterator(self, it: ndb.QueryIterator) -> None:
+    """
+    Update the current cursor from the value of the ndb.iterator passed in.
+
+    Args:
+      it: the iterator to take the cursor position from.
+    """
+    try:
+      self._ndb_cursor = typing.cast(ndb.Cursor, it.cursor_after())
+      self._cursor_state = _QueryCursorState.IN_PROGRESS
+    except ndb_exceptions.BadArgumentError:
+      # This exception can happen when iterator has not begun iterating
+      # and it.next() is the very first element.
+      #
+      # In those cases, `cursor_after()`` would not be 'after' any element,
+      # throwing this exception.
+
+      # We represent this by setting the state to STARTED.
+      self._ndb_cursor = None
+      self._cursor_state = _QueryCursorState.STARTED
+
+  @property
+  def ndb_cursor(self) -> ndb.Cursor | None:
+    """The inner ndb cursor, could be None"""
+    if self._cursor_state == _QueryCursorState.IN_PROGRESS:
+      return self._ndb_cursor
+
+    return None
+
+  @property
+  def ended(self) -> bool:
+    """
+    Whether the cursor has finished or not.
+    """
+    return self._cursor_state == _QueryCursorState.ENDED
+
+  def url_safe_encode(self) -> str | None:
+    """
+    Create a url safe page token to pass back to the API caller.
+    """
+    cursor_part: str = ''
+    match self._cursor_state:
+      case _QueryCursorState.STARTED:
+        cursor_part = _FIRST_PAGE_TOKEN
+      case _QueryCursorState.IN_PROGRESS:
+        # Assume that IN_PROGRESS means self.cursor is always set.
+        # Loudly throw an exception if this is not the case
+        cursor_part = self._ndb_cursor.urlsafe().decode()  # type: ignore
+      case _QueryCursorState.ENDED:
+        # If ENDED, we want to return None to not include
+        # a token in the response
+        return None
+
+    return str(self.query_number) + _METADATA_SEPARATOR + cursor_part

gcp/api/integration_tests.py

@@ -730,10 +730,11 @@ def test_query_batch(self):
             ]
         }, response.json())
 
-  @unittest.skip("Run this test locally with " +
-                 "MAX_VULN_LISTED_PRE_EXCEEDED at a lower value")
+  @unittest.skipIf(
+      os.getenv('LOW_MAX_THRESH', '0') != '1', "Run this test locally with " +
+      "MAX_VULN_LISTED_PRE_EXCEEDED at a lower value (around 10)")
   def test_query_pagination(self):
-    """Test query by package."""
+    """Test query by package with pagination."""
     response = requests.post(
         _api() + _BASE_QUERY,
         data=json.dumps(
@@ -763,9 +764,52 @@ def test_query_pagination(self):
 
     self.assertEqual(set(), vulns_first.intersection(vulns_second))
 
-  @unittest.skip("Run this test locally with " +
-                 "MAX_VULN_LISTED_PRE_EXCEEDED at a lower value")
-  def test_query_package_purl(self):
+  @unittest.skipIf(
+      os.getenv('LOW_MAX_THRESH', '0') != '1', "Run this test locally with " +
+      "MAX_VULN_LISTED_PRE_EXCEEDED at a lower value (around 10)")
+  def test_query_pagination_no_ecosystem(self):
+    """Test query with pagination but no ecosystem."""
+    response = requests.post(
+        _api() + _BASE_QUERY,
+        data=json.dumps({
+            'package': {
+                'name': 'django',
+            },
+            # Test with a version that is ambiguous whether it
+            # belongs to semver or generic version
+            'version': '5.0.1',
+        }),
+        timeout=_TIMEOUT)
+
+    result = response.json()
+    vulns_first = set(v['id'] for v in result['vulns'])
+    self.assertIn('next_page_token', result)
+    self.assertTrue(str.startswith(result['next_page_token'], '2:'))
+
+    response = requests.post(
+        _api() + _BASE_QUERY,
+        data=json.dumps({
+            'package': {
+                'name': 'django',
+            },
+            'version': '5.0.1',
+            'page_token': result['next_page_token'],
+        }),
+        timeout=_TIMEOUT)
+
+    result = response.json()
+    vulns_second = set(v['id'] for v in result['vulns'])
+
+    self.assertIn('next_page_token', result)
+    # There is not enough django vulns to simultaneously test multiple pages,
+    # and pass the other tests
+    # self.assertTrue(str.startswith(result['next_page_token'], '1:'))
+    self.assertEqual(set(), vulns_first.intersection(vulns_second))
+
+  @unittest.skipIf(
+      os.getenv('LOW_MAX_THRESH', '0') != '1', "Run this test locally with " +
+      "MAX_VULN_LISTED_PRE_EXCEEDED at a lower value (around 10)")
+  def test_query_package_purl_paging(self):
     """Test query by package (purl)."""
     response = requests.post(
         _api() + _BASE_QUERY,
@@ -890,18 +934,20 @@ def print_logs(filename):
 
 if __name__ == '__main__':
   if len(sys.argv) < 2:
-    print(f'Usage: {sys.argv[0]} path/to/credential.json')
+    print(
+        f'Usage: {sys.argv[0]} path/to/credential.json [...optional specific tests]'
+    )
     sys.exit(1)
 
   subprocess.run(
       ['docker', 'pull', 'gcr.io/endpoints-release/endpoints-runtime:2'],
       check=True)
 
-  credential_path = sys.argv.pop()
+  credential_path = sys.argv.pop(1)
   server = test_server.start(credential_path, port=_PORT)
-  time.sleep(30)
+  time.sleep(10)
 
   try:
-    unittest.main()
+    unittest.main(argv=sys.argv)
   finally:
     server.stop()