benchmark open and diagnostic

.github/workflows/benchmark.yaml

@@ -0,0 +1,55 @@
+name: Benchmark server
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: write
+
+jobs:
+  benchmark:
+    name: Run pytest-benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+      - name: Run benchmark
+        run: |
+          cd server
+          pip install -r requirements.txt -r test-requirements.txt
+          pytest buildoutls/bench/slapos.py --benchmark-json ../benchmark-data.json
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - name: Initialise base benchmark result (pull request)
+        run: |
+          ls ./cache
+          if test -f ./cache/benchmark-data-main.json ; then cp ./cache/benchmark-data-main.json ./cache/benchmark-data-pull-request.json ; fi
+        if: ${{ github.event_name == 'pull_request' }}
+      - name: Store benchmark result (pull request)
+        uses: benchmark-action/[email protected]
+        with:
+          name: Python Benchmark with pytest-benchmark
+          tool: "pytest"
+          output-file-path: benchmark-data.json
+          external-data-json-path: ./cache/benchmark-data-pull-request.json
+          alert-threshold: "110%"
+          fail-threshold: "120%"
+        if: ${{ github.event_name == 'pull_request' }}
+
+      - name: Store benchmark result (main)
+        uses: benchmark-action/[email protected]
+        with:
+          name: Python Benchmark with pytest-benchmark
+          tool: "pytest"
+          output-file-path: benchmark-data.json
+          external-data-json-path: ./cache/benchmark-data-main.json
+          alert-threshold: "110%"
+          fail-threshold: "120%"
+        if: ${{ github.event_name == 'push' }}

.vscode/settings.json

@@ -27,7 +27,7 @@
   "python.formatting.provider": "yapf",
   "python.linting.pylintEnabled": false,
   "python.linting.mypyEnabled": true,
-  "python.pythonPath": ".env/bin/python",
+  "python.pythonPath": "../venv/env/bin/python",
   "editor.codeActionsOnSave": {
     "source.fixAll.eslint": true
   }

server/buildoutls/bench/slapos.py

@@ -0,0 +1,132 @@
+import asyncio
+import pathlib
+import subprocess
+from typing import Any, List, no_type_check
+from unittest import mock
+
+import pytest
+from pygls.lsp.types import Diagnostic
+from pygls.server import LanguageServer
+from pygls.workspace import Workspace
+
+from ..buildout import (
+    _extends_dependency_graph,
+    _parse_cache,
+    _resolved_buildout_cache,
+    open,
+)
+from ..diagnostic import getDiagnostics
+
+
+@pytest.fixture
+def slapos_working_copy() -> pathlib.Path:
+  working_copy_path = pathlib.Path('.') / 'slapos'
+  if not working_copy_path.exists():
+    subprocess.check_call((
+        'git',
+        'clone',
+        '--depth=1',
+        '--branch=1.0.238',
+        'https://github.com/slapos/slapos',
+    ))
+  return working_copy_path.absolute()
+
+
+def clear_caches() -> None:
+  _resolved_buildout_cache.clear()
+  _parse_cache.clear()
+  _extends_dependency_graph.clear()
+
+
+@pytest.fixture
+def no_pypi_diagnostics() -> Any:
+  with mock.patch(
+        'buildoutls.diagnostic.pypi.PyPIClient.get_known_vulnerabilities', return_value=(),),\
+     mock.patch(
+       'buildoutls.diagnostic.pypi.PyPIClient.get_latest_version', return_value=None):
+    yield
+
+
+# https://github.com/ionelmc/pytest-benchmark/issues/66#issuecomment-575853801
+@no_type_check
+@pytest.fixture(scope='function')
+def aio_benchmark(benchmark):
+  import asyncio
+  import threading
+
+  class Sync2Async:
+    def __init__(self, coro, *args, **kwargs):
+      self.coro = coro
+      self.args = args
+      self.kwargs = kwargs
+      self.custom_loop = None
+      self.thread = None
+
+    def start_background_loop(self) -> None:
+      asyncio.set_event_loop(self.custom_loop)
+      self.custom_loop.run_forever()
+
+    def __call__(self):
+      evloop = None
+      awaitable = self.coro(*self.args, **self.kwargs)
+      try:
+        evloop = asyncio.get_running_loop()
+      except:
+        pass
+      if evloop is None:
+        return asyncio.run(awaitable)
+      else:
+        if not self.custom_loop or not self.thread or not self.thread.is_alive(
+        ):
+          self.custom_loop = asyncio.new_event_loop()
+          self.thread = threading.Thread(target=self.start_background_loop,
+                                         daemon=True)
+          self.thread.start()
+
+        return asyncio.run_coroutine_threadsafe(awaitable,
+                                                self.custom_loop).result()
+
+  def _wrapper(func, *args, **kwargs):
+    if asyncio.iscoroutinefunction(func):
+      benchmark(Sync2Async(func, *args, **kwargs))
+    else:
+      benchmark(func, *args, **kwargs)
+
+  return _wrapper
+
+
+@pytest.mark.parametrize('cache', ('with_cache', 'without_cache'))
+@pytest.mark.parametrize('profile_relative_path', (
+    'stack/slapos.cfg',
+    'stack/erp5/buildout.cfg',
+    'stack/erp5/rsyslogd.cfg.in',
+    'stack/erp5/instance.cfg.in',
+    'stack/erp5/instance-erp5.cfg.in',
+))
+async def test_open_and_diagnostic(
+    no_pypi_diagnostics: Any,
+    slapos_working_copy: pathlib.Path,
+    aio_benchmark: Any,
+    profile_relative_path: pathlib.Path,
+    cache: Any,
+) -> None:
+  doc_uri = (slapos_working_copy / profile_relative_path).as_uri()
+  workspace = Workspace(slapos_working_copy.as_uri())  # type: ignore
+  ls = LanguageServer(asyncio.new_event_loop())
+  ls.lsp.workspace = workspace
+
+  async def open_and_get_diagnostics() -> List[Diagnostic]:
+    diags: List[Diagnostic] = []
+    await open(ls, doc_uri)
+    async for diag in getDiagnostics(ls, doc_uri):
+      diags.append(diag)
+    return diags
+
+  # warmup
+  await open_and_get_diagnostics()
+
+  @aio_benchmark  # type: ignore
+  async def open_and_get_diagnostics_bench() -> None:
+    if cache == 'without_cache':
+      clear_caches()
+    await open_and_get_diagnostics()

server/setup.cfg

@@ -50,3 +50,5 @@ asyncio_mode = auto
 exclude_lines =
     pragma: no cover
     if TYPE_CHECKING:
+omit =
+    buildoutls/bench/*

server/test-requirements.txt

@@ -9,16 +9,18 @@ iniconfig==1.1.1
 isort==5.10.1
 lazy-object-proxy==1.7.1
 mccabe==0.7.0
-mypy==0.942
 mypy-extensions==0.4.3
+mypy==0.942
 platformdirs==2.5.1
 pluggy==0.13.1
+py-cpuinfo==8.0.0
 py==1.10.0
 pylint==2.13.1
 pyparsing==2.4.7
-pytest==7.1.1
 pytest-asyncio==0.18.3
+pytest-benchmark==3.4.1
 pytest-cov==3.0.0
+pytest==7.1.1
 requests==2.25.1
 responses==0.13.3
 six==1.16.0