From afe395f18807620682d11436372f882e19465a9e Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 19 Apr 2024 15:06:21 +0100 Subject: [PATCH 01/27] ci: remove warnings about outdated node versions Signed-off-by: Pau Ruiz Safont --- .github/workflows/1.249-lcm.yml | 41 ++++++++++++--------------------- .github/workflows/docs.yml | 2 +- .github/workflows/hugo.yml | 4 ++-- .github/workflows/main.yml | 2 +- .github/workflows/release.yml | 2 +- 5 files changed, 20 insertions(+), 31 deletions(-) diff --git a/.github/workflows/1.249-lcm.yml b/.github/workflows/1.249-lcm.yml index 31d5e0b49c8..39132476bd9 100644 --- a/.github/workflows/1.249-lcm.yml +++ b/.github/workflows/1.249-lcm.yml @@ -25,8 +25,6 @@ jobs: ocaml-test: name: Ocaml tests runs-on: ubuntu-20.04 - env: - package: "xapi-cli-protocol xapi-client xapi-consts xapi-database xapi-datamodel xapi-types xapi xe" steps: - name: Checkout code @@ -34,6 +32,10 @@ jobs: with: ref: '1.249-lcm' + - name: Free space + shell: bash + run: sudo rm -rf /usr/local/lib/android + - name: Pull configuration from xs-opam run: | curl --fail --silent https://raw.githubusercontent.com/xapi-project/xs-opam/release/yangtze/lcm/tools/xs-opam-ci.env | cut -f2 -d " " > .env @@ -42,37 +44,24 @@ jobs: id: dotenv uses: falti/dotenv-action@v1 - - name: Retrieve date for cache key (year-week) - id: cache-key - run: echo "date=$(/bin/date -u "+%Y%W")" >> $GITHUB_OUTPUT - shell: bash - - - name: Restore opam cache - id: opam-cache - uses: actions/cache@v4 - with: - path: "~/.opam" - # invalidate cache every week, gets built using a scheduled job - key: ${{ steps.cache-key.outputs.date }}-1.249 - - name: Update Ubuntu repositories + shell: bash run: sudo apt-get update - name: Use ocaml - uses: avsm/setup-ocaml@v1 + uses: ocaml/setup-ocaml@v2 with: - ocaml-version: ${{ steps.dotenv.outputs.ocaml_version_full }} - opam-repository: ${{ steps.dotenv.outputs.repository }} + ocaml-compiler: ${{ steps.dotenv.outputs.ocaml_version_full }} + opam-repositories: | + xs-opam: ${{ steps.dotenv.outputs.repository }} + dune-cache: true - name: Install dependencies - run: | - opam update - opam pin add . --no-action - opam depext -u ${{ env.package }} - opam upgrade - opam install ${{ env.package }} --deps-only --with-test -v + shell: bash + run: opam install . --deps-only --with-test -v - - name: Build + - name: Configure and build + shell: bash run: | opam exec -- ./configure opam exec -- make @@ -85,4 +74,4 @@ jobs: - name: Avoid built packages to appear in the cache # only packages in this repository follow a branch, the rest point # to a tag - run: opam uninstall ${{ env.package }} + run: opam pin list --short | xargs opam unpin diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0ceb3016ec7..096fe18227b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -43,7 +43,7 @@ jobs: opam exec -- dune exec ocaml/xapi-storage/generator/src/main.exe -- gen_markdown --path=$STORAGE_DOCDIR - name: Deploy xapi-storage docs - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: deploy_key: ${{ secrets.ACTIONS_STORAGE_DEPLOY_KEY }} publish_dir: ${{ env.STORAGE_DOCDIR }} diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 2d9bfe2bcb2..0b48e6c8a62 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Hugo - uses: peaceiris/actions-hugo@v2 + uses: peaceiris/actions-hugo@v3 with: hugo-version: '0.119.0' @@ -24,7 +24,7 @@ jobs: hugo --minify - name: Deploy - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: deploy_key: ${{ secrets.ACTIONS_DOCS_DEPLOY_KEY }} publish_dir: ./doc/public diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9db781fd626..40d9af83faf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,7 +37,7 @@ jobs: - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 name: Run pre-commit checks (no spaces at end of lines, etc) if: ${{ matrix.python-version != '2.7' }} with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c4d133d2fa1..1c3dca70fcd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v4 - name: Use python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" From c4ec76873f2fbc1d0a81d752413c084df486a30b Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Mon, 11 Mar 2024 12:41:50 +0100 Subject: [PATCH 02/27] pyproject.toml update settings for pytest etc for running CI locally Also disables pylint checks that are safe to disable - from #5490 and 5424. Co-authored-by: Pau Ruiz Safont Co-authored-by: Fei Su Co-authored-by: Stephen Cheng Signed-off-by: Bernhard Kaindl --- pyproject.toml | 171 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 169 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 505f16e430c..59100ec7ab9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ # https://packaging.python.org/en/latest/specifications/pyproject-toml/ [project] name = "xen-api" -requires-python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +requires-python = ">=3.6.*" license = {file = "LICENSE"} keywords = ["xen-project", "Xen", "hypervisor", "libraries"] maintainers = [ @@ -27,14 +27,72 @@ repository = "https://github.com/xapi-project/xen-api" [tool.black] line-length = 88 + +# ----------------------------------------------------------------------------- +# Coverage.py - https://coverage.readthedocs.io/en/coverage-5.5/config.html +# ----------------------------------------------------------------------------- + +[tool.coverage.report] +# Here, developers can configure which lines do not need to be covered by tests: +exclude_lines = [ + "pragma: no cover", # standard pragma for not covering a line or block + "if TYPE_CHECKING:", # imports for type checking only + "pass", + # Other specific lines that do not need to be covered, comment in which file: +] +# precision digits to use when reporting coverage (sub-percent-digits are not reported): +precision = 0 +# skip_covered: Skip reporting files with 100% coverage: +skip_covered = true + + +[tool.coverage.run] +# Default command line for "coverage run": Run pytest in non-verbose mode +command_line = "-m pytest -p no:logging -p no:warnings" +# Default data file for "coverage run": Store coverage data in .git/.coverage +data_file = ".git/.coverage" +# Default context for "coverage run": Use the name of the test function +dynamic_context = "test_function" +# Default omit patterns for "coverage run": Omit test files and test directories +omit = [ + "python3/bin/__init__.py", + "python3/packages/__init__.py", + # omit tests anything in a test directory (focus on the code) + "python3/tests", + "scripts/test_*.py", + # omit anything in a .local directory anywhere + "*/.local/*", + # omit everything in /usr + "/usr/*", +] +relative_files = true + + +# Default output when writing "coveragle xml" data. This needs to match what +# diff-cover and coverage upload to Codecov expect +[tool.coverage.xml] +output = ".git/coverage.xml" + + +# Default output directory for writing "coverage html" data. +# Create it outside the source tree to avoid cluttering the source tree +[tool.coverage.html] +directory = ".git/coverage_html" +show_contexts = true + + [tool.isort] line_length = 88 -py_version = 27 +py_version = 36 profile = "black" combine_as_imports = true ensure_newline_before_comments = false +# ----------------------------------------------------------------------------- +# Mypy static analysis - https://mypy.readthedocs.io/en/stable/config_file.html +# ----------------------------------------------------------------------------- + [tool.mypy] # Note mypy has no config setting for PYTHONPATH, so you need to call it with: # PYTHONPATH="scripts/examples/python:.:scripts:scripts/plugins:scripts/examples" @@ -59,6 +117,115 @@ disallow_any_explicit = false disallow_any_generics = true disallow_any_unimported = true disallow_subclassing_any = true +disable_error_code = ["import-untyped"] # XenAPI is not typed yet + + +[[tool.mypy.overrides]] +module = ["packages.observer"] +disable_error_code = [ + "arg-type", # mypy does not know that the Context class is actually a dict + "override", # Typing problem in the used library + "misc", + "no-any-unimported", +] + + +# ----------------------------------------------------------------------------- +# Pylint - https://pylint.pycqa.org/en/latest/technical_reference/features.html +# ----------------------------------------------------------------------------- + +[tool.pylint.design] +max-branches = 43 # perfmon has 43 branches in a function + + +[tool.pylint.messages_control] +# These are safe to disable, fixing them is best done during a later code cleanup phases +disable = [ + "broad-exception-caught", + "no-else-break", + "no-else-return", + "consider-using-f-string", # f-strings is the big new feature of Python 3.6, + "consider-using-with", # but like with, best done during code cleanup phase + "duplicate-code", # likewise. This is a code cleanup task + "import-error", # pylint does not do inter-procedural analysis + "invalid-name", # doesn't conform to snake_case naming style + "missing-function-docstring", # Best done in the code documentation phase + "missing-module-docstring", # Likewise, best done in the code documentation phase + "missing-class-docstring", # Likewise, best done in the code documentation phase + "no-member", # Existing code breaches this, not part of porting + "no-else-break", # else clause following a break statement + "protected-access", # Best done during the code cleanup phase + "super-with-arguments", # Consider using Python 3 style super(no args) calls + "too-many-branches", # Existing code breaches this, not part of porting + "too-many-arguments", # Likewise, not part of porting + "too-many-locals", # Likewise, not part of porting + "too-many-statements", # Likewise, not part of porting + "unnecessary-pass", # Cosmetic, best done during the code cleanup phase + "useless-object-inheritance", # Useless object inheritance from object, likewise +] + + +# ----------------------------------------------------------------------------- +# Pyright is the static analysis behind the VSCode Python extension / Pylance +# https://microsoft.github.io/pyright/#/configuration?id=main-configuration-options +# ----------------------------------------------------------------------------- + +[tool.pyright] +# Specifies the paths of directories or files that should be included in the +# analysis. If no paths are specified, all files in the workspace are included: +include = ["python3", "ocaml/xcp-rrdd"] + +# Conditionalize the stube files for type definitions based on the platform: +pythonPlatform = "Linux" + +# typeCheckingMode: "off", "basic", "standard" or "strict" +typeCheckingMode = "standard" + +# Specifies the version of Python that will be used to execute the source code. +# Generate errors if the source code makes use of language features that are +# not supported in that version. It will also tailor its use of type stub files, +# which conditionalizes type definitions based on the version. If no version is +# specified, pyright will use the version of the current python interpreter, +# if one is present: +pythonVersion = "3.6" + +# Paths of directories or files that should use "strict" analysis if they are +# included. This is the same as manually adding a "# pyright: strict" comment. +# In strict mode, most type-checking rules are enabled, and the type-checker +# will be more aggressive in inferring types. If no paths are specified, strict +# mode is not enabled: +strict = ["python3/tests/test_observer.py"] + +# +# Paths to exclude from analysis. If a file is excluded, it will not be +# analyzed. +# +# FIXME: Some of these may have type errors, so they should be inspected and fixed: +# +exclude = [ + "ocaml/xcp-rrdd/scripts/rrdd/rrdd.py", + "ocaml/xcp-rrdd/scripts/rrdd/rrdd-example.py", + "python3/packages/observer.py", + "python3/tests/pytype_reporter.py", +] + + +# ----------------------------------------------------------------------------- +# Pytest is the test framework, for discovering and running tests, fixtures etc +# https://pytest.readthedocs.io/en/latest/customize.html +# ----------------------------------------------------------------------------- + + +[tool.pytest.ini_options] +addopts = "-ra" # Show the output of all tests, including those that passed +log_cli = true # Capture log messages and show them in the output as well +log_cli_level = "INFO" +python_files = ["test_*.py", "it_*.py"] +python_functions = ["test_", "it_", "when_"] +pythonpath = "scripts/examples/python" # Allows to import the XenAPI module +required_plugins = ["pytest-mock"] +testpaths = ["python3", "scripts", "ocaml/xcp-rrdd"] +xfail_strict = true # is used to fail tests that are marked as xfail but pass(for TDD) [tool.pytype_reporter] From 06a3d67f9ab5bd4dc6990f9102ddd7a93506dec7 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Mon, 11 Mar 2024 12:19:48 +0100 Subject: [PATCH 03/27] pyproject.toml: Migrate pytype_reporter from scripts to python3 Signed-off-by: Bernhard Kaindl --- pyproject.toml | 59 ++++------------------------------------------ pytype_reporter.py | 12 ++++++---- 2 files changed, 12 insertions(+), 59 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 59100ec7ab9..d7bb0d7d1a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -236,65 +236,16 @@ discard_messages_matching = [ "No attribute 'group' on None", "No Node.TEXT_NODE in module xml.dom.minidom, referenced from 'xml.dom.expatbuilder'" ] -expected_to_fail = [ - "scripts/hfx_filename", - "scripts/perfmon", - # Need 2to3 -w and maybe a few other minor updates: - "scripts/hatests", - "scripts/backup-sr-metadata.py", - "scripts/restore-sr-metadata.py", - "scripts/nbd_client_manager.py", - # No attribute 'popen2' on module 'os' [module-attr] and a couple more: - "scripts/mail-alarm", - # SSLSocket.send() only accepts bytes, not unicode string as argument: - "scripts/examples/python/exportimport.py", - # Other fixes needed: - "scripts/examples/python/mini-xenrt.py", - "scripts/examples/python/XenAPI/XenAPI.py", - "scripts/examples/python/monitor-unwanted-domains.py", - "scripts/examples/python/shell.py", - "scripts/examples/smapiv2.py", - "scripts/static-vdis", - # add_interface: unsupported operand type(s) for +: str and UsbInterface - "scripts/usb_scan.py", - # TestUsbScan.assertIn() is called with wrong arguments(code not iterable) - "scripts/test_usb_scan.py", - "scripts/plugins/extauth-hook-AD.py", -] +expected_to_fail = [] [tool.pytype] inputs = [ - "scripts/hfx_filename", - "scripts/perfmon", - "scripts/static-vdis", - "scripts/Makefile", - "scripts/generate-iscsi-iqn", - "scripts/hatests", - "scripts/host-display", - "scripts/mail-alarm", - "scripts/print-custom-templates", - "scripts/probe-device-for-file", - "scripts/xe-reset-networking", - "scripts/xe-scsi-dev-map", - "scripts/examples/python", - "scripts/yum-plugins", - "scripts/*.py", - "python3/packages/*.py", - - # To be added later, - # when converted to Python3-compatible syntax: - # "ocaml/message-switch/python", - # "ocaml/idl/ocaml_backend/python", - # "ocaml/xapi-storage/python", + "python3/", + "ocaml/xcp-rrdd", ] disable = [ - "import-error", # xenfsimage, xcp.bootloader. xcp.cmd - "ignored-abstractmethod", - "ignored-metaclass", - # https://github.com/google/pytype/issues/1130, - # https://github.com/google/pytype/issues/1485: - "pyi-error", ] platform = "linux" -pythonpath = "scripts/examples/python:.:scripts:scripts/plugins:scripts/examples" +# Allow pytype to find the XenAPI module, the rrdd module and python3 modules: +pythonpath = "python3:scripts/examples/python:ocaml/xcp-rrdd/scripts/rrdd" diff --git a/pytype_reporter.py b/pytype_reporter.py index 8207053d7ec..877dc29c9d8 100755 --- a/pytype_reporter.py +++ b/pytype_reporter.py @@ -601,11 +601,13 @@ def main(): config.setdefault("expected_to_fail", []) debug("Expected to fail: %s", ", ".join(config["expected_to_fail"])) - changed_but_in_expected_to_fail = git_diff( - "--name-only", - find_branch_point(config), - *config["expected_to_fail"], - ).splitlines() + changed_but_in_expected_to_fail = [] + if config["expected_to_fail"] != []: + changed_but_in_expected_to_fail = git_diff( + "--name-only", + find_branch_point(config), + *config["expected_to_fail"], + ).splitlines() if check_only_reverts_from_branch_point(config, changed_but_in_expected_to_fail): return run_pytype_and_generate_summary(config) From 45ca37539da0369ea0cbf521778264a405cbd041 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 19 Apr 2024 16:26:20 +0100 Subject: [PATCH 04/27] ci: do not comment on PRs after merging The comments were far too long and most of the time did not provide any useful information Signed-off-by: Pau Ruiz Safont --- .github/workflows/main.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 40d9af83faf..095557f4a89 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -107,17 +107,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PYTYPE_REPORTER_DEBUG: True - # Try to add pytype_report.py's summary file as a comment to the PR: - # Documentation: https://github.com/marketplace/actions/add-pr-comment - - name: Add the pytype summary as a comment to the PR (if permitted) - uses: mshick/add-pr-comment@v2 - # Depends on pytype checks, which are not run for python 2.7: - if: ${{ matrix.python-version != '2.7' }} - # Fails for user workflows without permissions(fork-based pull requests): - continue-on-error: true - with: - message-path: .git/pytype-summary.md # Add the content of it as comment - ocaml-tests: name: Run OCaml tests runs-on: ubuntu-20.04 From 4661655f36189a9518019299d69867c316a8930e Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 19 Apr 2024 16:49:06 +0100 Subject: [PATCH 05/27] ci: ignore pylint and pyflakes checks This can happen when there are too many files to be tested, due to a bug Signed-off-by: Pau Ruiz Safont --- .github/workflows/main.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 095557f4a89..dde27bf2421 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,6 +98,7 @@ jobs: level: warning # To be customized to cover remaining Python scripts: glob_pattern: "**/*.py" + continue-on-error: true - name: Run pytype checks if: ${{ matrix.python-version != '2.7' }} @@ -107,6 +108,13 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PYTYPE_REPORTER_DEBUG: True + - name: pyflakes + uses: reviewdog/action-pyflakes@v1 + with: + github_token: ${{ secrets.github_token }} + continue-on-error: true + + ocaml-tests: name: Run OCaml tests runs-on: ubuntu-20.04 @@ -162,13 +170,6 @@ jobs: - name: quality-gate run: make quality-gate - - name: pyflakes - uses: reviewdog/action-pyflakes@master - with: - github_token: ${{ secrets.github_token }} - reporter: github-pr-review - level: error - test-sdk-builds: name: Test SDK builds uses: ./.github/workflows/generate-and-build-sdks.yml From 8244e3360c0c5bb2384576fff6e9bffa57bd847f Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Wed, 6 Mar 2024 12:00:00 +0100 Subject: [PATCH 06/27] test_observer.py: Add setUp() and tearDown() of mock modules Signed-off-by: Bernhard Kaindl --- python3/tests/test_observer.py | 44 +++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/python3/tests/test_observer.py b/python3/tests/test_observer.py index 53944d97ca9..088183d5375 100644 --- a/python3/tests/test_observer.py +++ b/python3/tests/test_observer.py @@ -12,17 +12,6 @@ mock_listdir.return_value = [] from packages import observer -# mock modules to avoid dependencies -sys.modules["opentelemetry"] = MagicMock() -sys.modules["opentelemetry.sdk.resources"] = MagicMock() -sys.modules["opentelemetry.sdk.trace"] = MagicMock() -sys.modules["opentelemetry.sdk.trace.export"] = MagicMock() -sys.modules["opentelemetry.exporter.zipkin.json"] = MagicMock() -sys.modules["opentelemetry.baggage.propagation"] = MagicMock() -sys.modules["opentelemetry.trace.propagation.tracecontext"] = MagicMock() -sys.modules["opentelemetry.context"] = MagicMock() -sys.modules["opentelemetry.trace"] = MagicMock() - TEST_CONFIG = """ XS_EXPORTER_BUGTOOL_ENDPOINT='/var/log/dt/test' OTEL_SERVICE_NAME='test-observer' @@ -31,11 +20,44 @@ TEST_OBSERVER_CONF = "test-observer.conf" OBSERVER_OPEN = "packages.observer.open" +# +# These are the modules that are mocked to avoid dependencies. +# Note: wrapt is not mocked: It is used to wrap the traced script. +# These modules are not imported at the top of observer.py, but are +# imported inside the observer._init_tracing(). This is why they are mocked +# in the test class before calling observer._init_tracing() and then deleted +# in the tearDown of the test class to avoid affecting other tests. +# +MOCKED_MODULES = [ + "opentelemetry", + "opentelemetry.sdk.resources", + "opentelemetry.sdk.trace", + "opentelemetry.sdk.trace.export", + "opentelemetry.exporter.zipkin.json", + "opentelemetry.baggage.propagation", + "opentelemetry.trace.propagation.tracecontext", + "opentelemetry.context", + "opentelemetry.trace", +] + # pylint: disable=missing-function-docstring,protected-access class TestObserver(unittest.TestCase): """Test python3/packages/observer.py""" + def setUp(self) -> None: + # As setup for this class, mock modules to avoid dependencies + for mock in MOCKED_MODULES: + sys.modules[mock] = MagicMock() + return super().setUp() + + def tearDown(self) -> None: + # On teardown, delete mocks so they do not affect other tests + # Otherwise, the mocks will be used in other tests and cause errors + for mock in MOCKED_MODULES: + del sys.modules[mock] + return super().tearDown() + def simple_method(self): """A simple helper method for tests to wrap using observer.span""" return 5 From 4cdcf7482aef07f43b12b3bd503cdba7d2e456f0 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Wed, 6 Mar 2024 12:00:00 +0100 Subject: [PATCH 07/27] observer.py: Update error handling Signed-off-by: Bernhard Kaindl --- python3/packages/observer.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python3/packages/observer.py b/python3/packages/observer.py index b257aa9b3f0..3419c89cd36 100644 --- a/python3/packages/observer.py +++ b/python3/packages/observer.py @@ -401,18 +401,25 @@ def run(file): return 0 except FileNotFoundError as e: print( - f"{__file__}: {' '.join(sys.argv)}\n{e.filename}: No such file", + f"{__file__} {' '.join(sys.argv)}:\nScript not found: {e.filename}", file=sys.stderr, ) return 2 - except Exception: - print( - f"{__file__}: {' '.join(sys.argv)}\n{traceback.format_exc()}", - file=sys.stderr) + except Exception as e: + print(f"{__file__} {' '.join(sys.argv)}:", file=sys.stderr) # the command + print("Exception in the traced script:", file=sys.stderr) + print(e, file=sys.stderr) # Print the exception message + print(traceback.format_exc(), file=sys.stderr) # Print the traceback return 139 # This is what the default SIGSEGV handler on Linux returns return run(argv0) if __name__ == "__main__": - sys.exit(main()) + # Only use sys.exit(ret) raising SystemExit() if the return code is not 0 + # to allow test_observer_as_script() to get the globals of observer.py: + + exit_code = main() # pylint: disable=invalid-name + logging.shutdown() # Reduces the unclosed socket warnings by PYTHONDEVMODE=yes + if exit_code: + sys.exit(exit_code) From e677c054b130fd36ec81928228b981f231724b6e Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 22 Apr 2024 13:03:57 +0100 Subject: [PATCH 08/27] ci: install observer.py dependencies Signed-off-by: Pau Ruiz Safont --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dde27bf2421..42c06764498 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,7 +51,7 @@ jobs: - name: Install dependencies only needed for python 3 if: ${{ matrix.python-version != '2.7' }} - run: pip install pandas pytype toml wrapt + run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt - name: Install common dependencies for Python ${{matrix.python-version}} run: pip install future mock pytest-coverage pytest-mock From 026818b317205c7c04ae427d29f307d0084ab1b0 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 22 Apr 2024 14:45:57 +0100 Subject: [PATCH 09/27] opam: delete xapi-stdext package It only existed because of dune limitations when it was in its own repository Signed-off-by: Pau Ruiz Safont --- Makefile | 4 ++-- dune-project | 15 --------------- xapi-stdext.opam | 34 ---------------------------------- 3 files changed, 2 insertions(+), 51 deletions(-) delete mode 100644 xapi-stdext.opam diff --git a/Makefile b/Makefile index 43ebc394d85..897862dab52 100644 --- a/Makefile +++ b/Makefile @@ -255,7 +255,7 @@ install: build doc sdk doc-json message-switch-unix xapi-idl forkexec xapi-forkexecd xapi-storage xapi-storage-script xapi-storage-cli \ xapi-nbd varstored-guard xapi-log xapi-open-uri xapi-tracing xapi-expiry-alerts cohttp-posix \ xapi-rrd xapi-inventory \ - xapi-stdext-date xapi-stdext-encodings xapi-stdext-pervasives xapi-stdext-std xapi-stdext-threads xapi-stdext-unix xapi-stdext-zerocheck xapi-stdext + xapi-stdext-date xapi-stdext-encodings xapi-stdext-pervasives xapi-stdext-std xapi-stdext-threads xapi-stdext-unix xapi-stdext-zerocheck # docs mkdir -p $(DESTDIR)$(DOCDIR) cp -r $(XAPIDOC)/jekyll $(DESTDIR)$(DOCDIR) @@ -277,7 +277,7 @@ uninstall: message-switch-unix xapi-idl forkexec xapi-forkexecd xapi-storage xapi-storage-script xapi-log \ xapi-open-uri xapi-tracing xapi-expiry-alerts cohttp-posix \ xapi-rrd xapi-inventory \ - xapi-stdext-date xapi-stdext-encodings xapi-stdext-pervasives xapi-stdext-std xapi-stdext-threads xapi-stdext-unix xapi-stdext-zerocheck xapi-stdext + xapi-stdext-date xapi-stdext-encodings xapi-stdext-pervasives xapi-stdext-std xapi-stdext-threads xapi-stdext-unix xapi-stdext-zerocheck compile_flags.txt: Makefile (ocamlc -config-var ocamlc_cflags;\ diff --git a/dune-project b/dune-project index 747fc62b133..dba83ffab8b 100644 --- a/dune-project +++ b/dune-project @@ -262,21 +262,6 @@ (name xapi-inventory) ) -(package - (name xapi-stdext) - (synopsis "Xapi's standard library extension") - (description "Dummy package that enables the usage of dune-release") - (depends - (xapi-stdext-date (= :version)) - (xapi-stdext-encodings (= :version)) - (xapi-stdext-pervasives (= :version)) - (xapi-stdext-std (= :version)) - (xapi-stdext-threads (= :version)) - (xapi-stdext-unix (= :version)) - (xapi-stdext-zerocheck (= :version)) - ) -) - (package (name xapi-stdext-date) (synopsis "Xapi's standard library extension, Dates") diff --git a/xapi-stdext.opam b/xapi-stdext.opam deleted file mode 100644 index e2654f782ab..00000000000 --- a/xapi-stdext.opam +++ /dev/null @@ -1,34 +0,0 @@ -# This file is generated by dune, edit dune-project instead -opam-version: "2.0" -synopsis: "Xapi's standard library extension" -description: "Dummy package that enables the usage of dune-release" -maintainer: ["Xapi project maintainers"] -authors: ["xen-api@lists.xen.org"] -license: "LGPL-2.1-only WITH OCaml-LGPL-linking-exception" -homepage: "https://xapi-project.github.io/" -bug-reports: "https://github.com/xapi-project/xen-api/issues" -depends: [ - "dune" {>= "2.0"} - "xapi-stdext-date" {= version} - "xapi-stdext-encodings" {= version} - "xapi-stdext-pervasives" {= version} - "xapi-stdext-std" {= version} - "xapi-stdext-threads" {= version} - "xapi-stdext-unix" {= version} - "xapi-stdext-zerocheck" {= version} -] -build: [ - ["dune" "subst"] {pinned} - [ - "dune" - "build" - "-p" - name - "-j" - jobs - "@install" - "@runtest" {with-test} - "@doc" {with-doc} - ] -] -dev-repo: "git+https://github.com/xapi-project/xen-api.git" From c59e4ff711ff98e50a5baec7c124019f453623b6 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 22 Apr 2024 14:47:42 +0100 Subject: [PATCH 10/27] opam: fix xapi-squeezed metadata Signed-off-by: Pau Ruiz Safont --- xapi-squeezed.opam | 1 - xapi-squeezed.opam.template | 1 - 2 files changed, 2 deletions(-) diff --git a/xapi-squeezed.opam b/xapi-squeezed.opam index 978e98953f7..52dd6fdc3dc 100644 --- a/xapi-squeezed.opam +++ b/xapi-squeezed.opam @@ -2,7 +2,6 @@ authors: ["xen-api@lists.xen.org"] license: "LGPL-2.1-only WITH OCaml-LGPL-linking-exception" opam-version: "2.0" -author: "dave.scott@eu.citrix.com" maintainer: "xen-api@lists.xen.org" homepage: "https://github.com/xapi-project/xen-api" bug-reports: "https://github.com/xapi-project/xen-api/issues" diff --git a/xapi-squeezed.opam.template b/xapi-squeezed.opam.template index fc590e1df24..84ad0840a82 100644 --- a/xapi-squeezed.opam.template +++ b/xapi-squeezed.opam.template @@ -1,5 +1,4 @@ opam-version: "2.0" -author: "dave.scott@eu.citrix.com" maintainer: "xen-api@lists.xen.org" homepage: "https://github.com/xapi-project/xen-api" bug-reports: "https://github.com/xapi-project/xen-api/issues" From 13eb35a843924f7b0b9085fa2c16c3ed637116b7 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 22 Apr 2024 15:27:48 +0100 Subject: [PATCH 11/27] opam: create package xapi-tracing-export This avoids circular dependencies Signed-off-by: Pau Ruiz Safont --- dune-project | 4 ++++ ocaml/libs/tracing/dune | 2 +- ocaml/xenopsd/lib/dune | 2 +- xapi-tracing-export.opam | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 xapi-tracing-export.opam diff --git a/dune-project b/dune-project index dba83ffab8b..20058ef9e72 100644 --- a/dune-project +++ b/dune-project @@ -66,6 +66,10 @@ (name xapi-tracing) ) +(package + (name xapi-tracing-export) +) + (package (name xapi-storage-script) ) diff --git a/ocaml/libs/tracing/dune b/ocaml/libs/tracing/dune index 6c18e2f15c1..0e1160818c2 100644 --- a/ocaml/libs/tracing/dune +++ b/ocaml/libs/tracing/dune @@ -7,7 +7,7 @@ (library (name tracing_export) (modules tracing_export) - (public_name xapi-tracing.export) + (public_name xapi-tracing-export) (libraries cohttp cohttp-posix diff --git a/ocaml/xenopsd/lib/dune b/ocaml/xenopsd/lib/dune index 0521c4b3db4..9f0a63e064c 100644 --- a/ocaml/xenopsd/lib/dune +++ b/ocaml/xenopsd/lib/dune @@ -42,7 +42,7 @@ xapi-stdext-threads xapi-stdext-unix xapi-tracing - xapi-tracing.export + xapi-tracing-export xmlm ) (preprocess diff --git a/xapi-tracing-export.opam b/xapi-tracing-export.opam new file mode 100644 index 00000000000..c58065eb2a3 --- /dev/null +++ b/xapi-tracing-export.opam @@ -0,0 +1,32 @@ +# This file is generated by dune, edit dune-project instead +license: "LGPL-2.1-only WITH OCaml-LGPL-linking-exception" +opam-version: "2.0" +maintainer: "xen-api@lists.xen.org" +authors: "xen-api@lists.xen.org" +homepage: "https://xapi-project.github.io/" +bug-reports: "https://github.com/xapi-project/xen-api.git" +dev-repo: "git+https://github.com/xapi-project/xen-api.git" +build: [[ "dune" "build" "-p" name "-j" jobs ]] + +available: [ os = "linux" ] +depends: [ + "ocaml" + "cohttp-posix" + "dune" + "cohttp" + "rpclib" + "xapi-log" + "xapi-open-uri" + "xapi-stdext-threads" + "xapi-stdext-unix" + "xapi-tracing" + "zstd" +] +synopsis: "Library required by xapi" +description: """ +These libraries are provided for backwards compatibility only. +No new code should use these libraries.""" +url { + src: + "https://github.com/xapi-project/xen-api/archive/master.tar.gz" +} From 95e83078a8e09626297eb7638ab961cfd31f4357 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 22 Apr 2024 15:31:05 +0100 Subject: [PATCH 12/27] datamodel_lifecycle: bump Signed-off-by: Pau Ruiz Safont --- ocaml/idl/datamodel_lifecycle.ml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ocaml/idl/datamodel_lifecycle.ml b/ocaml/idl/datamodel_lifecycle.ml index 28ff931ec59..94870d262c7 100644 --- a/ocaml/idl/datamodel_lifecycle.ml +++ b/ocaml/idl/datamodel_lifecycle.ml @@ -109,6 +109,12 @@ let prototyped_of_message = function Some "22.20.0" | "Repository", "set_gpgkey_path" -> Some "22.12.0" + | "PCI", "get_dom0_access_status" -> + Some "24.13.0-next" + | "PCI", "enable_dom0_access" -> + Some "24.13.0-next" + | "PCI", "disable_dom0_access" -> + Some "24.13.0-next" | "message", "destroy_many" -> Some "22.19.0" | "VTPM", "set_contents" -> From e699b39a8a73ed2f1c8e90a4de389a0f55e41de5 Mon Sep 17 00:00:00 2001 From: Lin Liu Date: Mon, 22 Apr 2024 06:42:20 +0000 Subject: [PATCH 13/27] CA-391859: Failed to stop varstord-guard varstord-guard has following configuration in the service file After=message-switch.service syslog.target This means varstored-guard needs to be stopped before message-switch, otherwise, it will hung and finally be killed by systemd during xe-toolstack-restart Move message-switch to the begining of the list to shutdown last as all other daemons depend on it Note: the list order is reversed by `for svc in $SERVICES` Signed-off-by: Lin Liu --- scripts/xe-toolstack-restart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/xe-toolstack-restart b/scripts/xe-toolstack-restart index 7cafee29fa4..32ee88609c5 100755 --- a/scripts/xe-toolstack-restart +++ b/scripts/xe-toolstack-restart @@ -27,10 +27,10 @@ echo "Executing $FILENAME" POOLCONF=`cat @ETCXENDIR@/pool.conf` if [ $POOLCONF == "master" ]; then MPATHALERT="mpathalert"; else MPATHALERT=""; fi -SERVICES="perfmon v6d xenopsd xenopsd-xc xenopsd-xenlight +SERVICES="message-switch perfmon v6d xenopsd xenopsd-xc xenopsd-xenlight xenopsd-simulator xenopsd-libvirt xcp-rrdd-iostat xcp-rrdd-squeezed xcp-rrdd-xenpm xcp-rrdd-gpumon xcp-rrdd xcp-networkd squeezed forkexecd - $MPATHALERT xapi-storage-script xapi-clusterd varstored-guard message-switch" + $MPATHALERT xapi-storage-script xapi-clusterd varstored-guard" tmp_file=$(mktemp --suffix="xe-toolstack-restart") systemctl stop stunnel@xapi > $tmp_file 2>&1 From 4c8fcdc3f413c4dcfae9e4c0ff9892f47d5e17fa Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Tue, 23 Apr 2024 20:35:15 +0100 Subject: [PATCH 14/27] Exposed methods to fetch the methods available in the API. Signed-off-by: Konstantina Chremmou --- ocaml/sdk-gen/csharp/autogen/src/Session.cs | 5 +++++ ocaml/sdk-gen/csharp/templates/JsonRpcClient.mustache | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/ocaml/sdk-gen/csharp/autogen/src/Session.cs b/ocaml/sdk-gen/csharp/autogen/src/Session.cs index a2aef1d672e..4a84b5bcd0c 100644 --- a/ocaml/sdk-gen/csharp/autogen/src/Session.cs +++ b/ocaml/sdk-gen/csharp/autogen/src/Session.cs @@ -333,6 +333,11 @@ public Dictionary RequestHeaders #endregion + public string[] GetSystemMethods() + { + return JsonRpcClient.system_list_methods(); + } + public static Session get_record(Session session, string _session) { Session newSession = new Session(session.Url) { opaque_ref = _session }; diff --git a/ocaml/sdk-gen/csharp/templates/JsonRpcClient.mustache b/ocaml/sdk-gen/csharp/templates/JsonRpcClient.mustache index ac3d0ca625c..307e02560f4 100644 --- a/ocaml/sdk-gen/csharp/templates/JsonRpcClient.mustache +++ b/ocaml/sdk-gen/csharp/templates/JsonRpcClient.mustache @@ -38,6 +38,13 @@ namespace XenAPI { public partial class JsonRpcClient { + public string[] system_list_methods() + { + var converters = new List { }; + var serializer = CreateSerializer(converters); + return Rpc("system.listMethods", new JArray(), serializer); + } + public Event event_get_record(string session, string _event) { var converters = new List {}; From e78773ed43b309b28b0f1a5b1a03497015073a11 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Tue, 23 Apr 2024 21:08:58 +0100 Subject: [PATCH 15/27] The github workflow artifacts for C contained unnecessary files. Signed-off-by: Konstantina Chremmou --- .github/workflows/generate-and-build-sdks.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index db28438062f..1886c33c368 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -28,7 +28,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: SDK_Source_C - path: _build/install/default/xapi/sdk/c/* + path: | + _build/install/default/xapi/sdk/c/* + !_build/install/default/xapi/sdk/c/dune - name: Store C# SDK source uses: actions/upload-artifact@v4 @@ -67,7 +69,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: SDK_Artifacts_C - path: source/* + path: | + source/* + !source/src/*.o build-csharp-sdk: name: Build C# SDK From cc55dd2d5afd275b84552e3e22b638219028a62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 09:18:05 +0100 Subject: [PATCH 16/27] CI: update to Ubuntu 22.04 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need to explicitly install python2, since it is no longer the default. Eventually we may want to drop this completely, but we're not there yet. Signed-off-by: Edwin Török --- .github/workflows/main.yml | 4 ++-- .github/workflows/setup-xapi-environment/action.yml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42c06764498..9bb4aee895d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,7 +117,7 @@ jobs: ocaml-tests: name: Run OCaml tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: # Ensure you also update test-sdk-builds # when changing this value, to keep builds @@ -158,7 +158,7 @@ jobs: deprecation-test: name: Deprecation tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout code diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index e25e0e184fb..c3785924e76 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -31,6 +31,10 @@ runs: shell: bash run: sudo apt-get update + - name: Install python2 + shell: bash + run: sudo apt-get install python2 + - name: Use disk with more space for TMPDIR and XDG_CACHE_HOME shell: bash run: | From c5e07819129c59bf17e7a4f01aa464f48e33e4c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 09:46:08 +0100 Subject: [PATCH 17/27] ci(nopin): pinning is very slow and not necessary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We do not publish or use the opam packages from this repo, and therefore pinning is unnecessary. 'opam install --deps-only --with-test .' works without pinning on the version of 'opam' we have here. Pinning ~71 packages is very slow since it makes a full copy of the source+.git folder every time. Skip pinning, this also saves a few gigabytes of disk space. This also makes the cleanup-xapi-environment action a no-op, so remove it. (in fact it'd fail because the output of opam list with master pins is empty). For testing the correctness of opam packages: * we still have the xs-opam CI * we can introduce opam-dune-lint that statically checks opam and dune for missing dependencies (needs new release first) This reduces CI times: * Run OCaml tests: from 15m24s to 10m16s * Setup XenAPI environment: from 7m1s to 5m55s Signed-off-by: Edwin Török --- .../workflows/cleanup-xapi-environment/action.yml | 13 ------------- .github/workflows/generate-and-build-sdks.yml | 3 --- .github/workflows/main.yml | 3 --- .github/workflows/setup-xapi-environment/action.yml | 1 + 4 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 .github/workflows/cleanup-xapi-environment/action.yml diff --git a/.github/workflows/cleanup-xapi-environment/action.yml b/.github/workflows/cleanup-xapi-environment/action.yml deleted file mode 100644 index 96323007e4e..00000000000 --- a/.github/workflows/cleanup-xapi-environment/action.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Cleanup XenAPI environment -description: Cleanup XenAPI environment created using the setup-xapi-environment composite action - -runs: - using: "composite" - steps: - - name: Uninstall unversioned packages and remove pins - shell: bash - # This should purge them from the cache, unversioned package have - # 'master' as its version - run: | - opam list | awk -F " " '$2 == "master" { print $1 }' | xargs opam uninstall - opam pin list | cut -f1 -d "." | xargs opam unpin diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index db28438062f..55ac3f4f09a 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -42,9 +42,6 @@ jobs: name: SDK_Source_PowerShell path: _build/install/default/xapi/sdk/powershell/* - - name: Cleanup XenAPI environment - uses: ./.github/workflows/cleanup-xapi-environment - build-c-sdk: name: Build C SDK runs-on: ubuntu-latest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9bb4aee895d..ada260060ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -153,9 +153,6 @@ jobs: - name: Check disk space run: df -h || true - - name: Cleanup XenAPI environment - uses: ./.github/workflows/cleanup-xapi-environment - deprecation-test: name: Deprecation tests runs-on: ubuntu-22.04 diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index c3785924e76..505032095ae 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -52,6 +52,7 @@ runs: ocaml-compiler: ${{ steps.dotenv.outputs.ocaml_version_full }} opam-repositories: | xs-opam: ${{ steps.dotenv.outputs.repository }} + opam-pin: false dune-cache: true - name: Install dependencies From ce2f58870cf783cf24a5d356e84f22658b9ea600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 10:26:15 +0100 Subject: [PATCH 18/27] ci(opam-dune-cache): cache dune builds from opam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting DUNE_CACHE_STORAGE_MODE=copy is needed during 'opam install', otherwise dune builds inside opam packages won't be cached. The sandbox used by opam is another partition/device and hardlinking fails with EXDEV. Signed-off-by: Edwin Török --- .github/workflows/setup-xapi-environment/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index 505032095ae..b703d523340 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -46,6 +46,8 @@ runs: echo "TMPDIR=${TMPDIR}" >>"$GITHUB_ENV" echo "XDG_CACHE_HOME=${XDG_CACHE_HOME}" >>"$GITHUB_ENV" + # We set DUNE_CACHE_STORAGE_MODE, it is required for dune cache to work inside opam for now, + # otherwise it gets EXDEV and considers it a cache miss - name: Use ocaml uses: ocaml/setup-ocaml@v2 with: @@ -54,6 +56,8 @@ runs: xs-opam: ${{ steps.dotenv.outputs.repository }} opam-pin: false dune-cache: true + env: + DUNE_CACHE_STORAGE_MODE: copy - name: Install dependencies shell: bash From 342e125728fff5b4ab845dc50db02f996c97ab95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 10:29:50 +0100 Subject: [PATCH 19/27] ci(norm): we have enough space now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we no longer pin the local opam packages we have a few more GiB of disk space, and we don't need to rm pre-existing files. This speeds up this step a little. Signed-off-by: Edwin Török --- .github/workflows/setup-xapi-environment/action.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index b703d523340..d46ae3a5b96 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -8,10 +8,6 @@ inputs: runs: using: "composite" steps: - - name: Free space - shell: bash - run: sudo rm -rf /usr/local/lib/android - - name: Pull configuration from xs-opam shell: bash run: | From 3907e2188d4644fbe994ce970182240b87ba10ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 10:44:29 +0100 Subject: [PATCH 20/27] ci: separate workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dune cache inside 'setup-ocaml' is trimmed to 5GB/jobcount, which is currently about 10. But not all jobs here use the dune cache, so separate the workflows. (Also the dune cache will get compressed afterwards). The SDK build does use the dune cache, but eventually it might become smaller, e.g. if we change it to install just the dependencies needed for SDK gen, and not everything. It does have a lot of unrelated jobs though. The workflows need a unique name, otherwise they cancel each-other. The dune cache is ~1.4GB now compressed, which should speed up the setup-ocaml step on next run as more dependencies will be cached. Setup XenAPI environment now only takes 2m51s instead of 5m55s. Signed-off-by: Edwin Török --- .github/workflows/main.yml | 126 -------------------------------- .github/workflows/other.yml | 138 ++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/other.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ada260060ad..5f2dd84a112 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,109 +12,6 @@ concurrency: # On new push, cancel old workflows from the same PR, branch or tag cancel-in-progress: true jobs: - python-test: - name: Python tests - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python-version: ["2.7", "3.11"] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # To check which files changed: origin/master..HEAD - - uses: LizardByte/setup-python-action@master - with: - python-version: ${{matrix.python-version}} - - - uses: actions/cache@v4 - name: Setup cache for running pre-commit fast - with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" - name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" - - - uses: pre-commit/action@v3.0.1 - name: Run pre-commit checks (no spaces at end of lines, etc) - if: ${{ matrix.python-version != '2.7' }} - with: - extra_args: --all-files --verbose --hook-stage commit - env: - SKIP: no-commit-to-branch - - - name: Install dependencies only needed for python 2 - if: ${{ matrix.python-version == '2.7' }} - run: pip install enum - - - name: Install dependencies only needed for python 3 - if: ${{ matrix.python-version != '2.7' }} - run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt - - - name: Install common dependencies for Python ${{matrix.python-version}} - run: pip install future mock pytest-coverage pytest-mock - - - name: Run Pytest for python 2 and get code coverage for Codecov - if: ${{ matrix.python-version == '2.7' }} - run: > - pytest - --cov=scripts --cov=ocaml/xcp-rrdd - scripts/ ocaml/xcp-rrdd -vv -rA - --junitxml=.git/pytest${{matrix.python-version}}.xml - --cov-report term-missing - --cov-report xml:.git/coverage${{matrix.python-version}}.xml - env: - PYTHONDEVMODE: yes - - - name: Run Pytest for python 3 and get code coverage for Codecov - if: ${{ matrix.python-version != '2.7' }} - run: > - pytest - --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/ - scripts/ ocaml/xcp-rrdd python3/ -vv -rA - --junitxml=.git/pytest${{matrix.python-version}}.xml - --cov-report term-missing - --cov-report xml:.git/coverage${{matrix.python-version}}.xml - env: - PYTHONDEVMODE: yes - - - name: Upload Python ${{matrix.python-version}} coverage report to Codecov - uses: codecov/codecov-action@v3 - with: - directory: .git - files: coverage${{matrix.python-version}}.xml - env_vars: OS,PYTHON - fail_ci_if_error: false - flags: python${{matrix.python-version}} - name: coverage${{matrix.python-version}} - verbose: true - - - uses: dciborow/action-pylint@0.1.0 - if: ${{ matrix.python-version != '2.7' }} - with: - reporter: github-pr-review - level: warning - # To be customized to cover remaining Python scripts: - glob_pattern: "**/*.py" - continue-on-error: true - - - name: Run pytype checks - if: ${{ matrix.python-version != '2.7' }} - run: ./pytype_reporter.py - env: - PR_NUMBER: ${{ github.event.number }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYTYPE_REPORTER_DEBUG: True - - - name: pyflakes - uses: reviewdog/action-pyflakes@v1 - with: - github_token: ${{ secrets.github_token }} - continue-on-error: true - - ocaml-tests: name: Run OCaml tests runs-on: ubuntu-22.04 @@ -152,26 +49,3 @@ jobs: - name: Check disk space run: df -h || true - - deprecation-test: - name: Deprecation tests - runs-on: ubuntu-22.04 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Generate empty configuration for make to be happy - run: touch config.mk - - - name: quality-gate - run: make quality-gate - - test-sdk-builds: - name: Test SDK builds - uses: ./.github/workflows/generate-and-build-sdks.yml - with: - # Ensure you also update ocaml-tests - # when changing this value, to keep builds - # consistent - xapi_version: "v0.0.0" diff --git a/.github/workflows/other.yml b/.github/workflows/other.yml new file mode 100644 index 00000000000..d65b7abe575 --- /dev/null +++ b/.github/workflows/other.yml @@ -0,0 +1,138 @@ +name: Build and test (other) + +on: + push: + pull_request: + schedule: + # run daily, this refreshes the cache + - cron: "13 2 * * *" + +concurrency: # On new push, cancel old workflows from the same PR, branch or tag: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + python-test: + name: Python tests + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python-version: ["2.7", "3.11"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # To check which files changed: origin/master..HEAD + - uses: LizardByte/setup-python-action@master + with: + python-version: ${{matrix.python-version}} + + - uses: actions/cache@v4 + name: Setup cache for running pre-commit fast + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" + name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" + + - uses: pre-commit/action@v3.0.1 + name: Run pre-commit checks (no spaces at end of lines, etc) + if: ${{ matrix.python-version != '2.7' }} + with: + extra_args: --all-files --verbose --hook-stage commit + env: + SKIP: no-commit-to-branch + + - name: Install dependencies only needed for python 2 + if: ${{ matrix.python-version == '2.7' }} + run: pip install enum + + - name: Install dependencies only needed for python 3 + if: ${{ matrix.python-version != '2.7' }} + run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt + + - name: Install common dependencies for Python ${{matrix.python-version}} + run: pip install future mock pytest-coverage pytest-mock + + - name: Run Pytest for python 2 and get code coverage for Codecov + if: ${{ matrix.python-version == '2.7' }} + run: > + pytest + --cov=scripts --cov=ocaml/xcp-rrdd + scripts/ ocaml/xcp-rrdd -vv -rA + --junitxml=.git/pytest${{matrix.python-version}}.xml + --cov-report term-missing + --cov-report xml:.git/coverage${{matrix.python-version}}.xml + env: + PYTHONDEVMODE: yes + + - name: Run Pytest for python 3 and get code coverage for Codecov + if: ${{ matrix.python-version != '2.7' }} + run: > + pytest + --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/ + scripts/ ocaml/xcp-rrdd python3/ -vv -rA + --junitxml=.git/pytest${{matrix.python-version}}.xml + --cov-report term-missing + --cov-report xml:.git/coverage${{matrix.python-version}}.xml + env: + PYTHONDEVMODE: yes + + - name: Upload Python ${{matrix.python-version}} coverage report to Codecov + uses: codecov/codecov-action@v3 + with: + directory: .git + files: coverage${{matrix.python-version}}.xml + env_vars: OS,PYTHON + fail_ci_if_error: false + flags: python${{matrix.python-version}} + name: coverage${{matrix.python-version}} + verbose: true + + - uses: dciborow/action-pylint@0.1.0 + if: ${{ matrix.python-version != '2.7' }} + with: + reporter: github-pr-review + level: warning + # To be customized to cover remaining Python scripts: + glob_pattern: "**/*.py" + continue-on-error: true + + - name: Run pytype checks + if: ${{ matrix.python-version != '2.7' }} + run: ./pytype_reporter.py + env: + PR_NUMBER: ${{ github.event.number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYTYPE_REPORTER_DEBUG: True + + - name: pyflakes + uses: reviewdog/action-pyflakes@v1 + with: + github_token: ${{ secrets.github_token }} + continue-on-error: true + + deprecation-test: + name: Deprecation tests + runs-on: ubuntu-22.04 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Generate empty configuration for make to be happy + run: touch config.mk + + - name: quality-gate + run: make quality-gate + + test-sdk-builds: + name: Test SDK builds + uses: ./.github/workflows/generate-and-build-sdks.yml + with: + # Ensure you also update ocaml-tests + # when changing this value, to keep builds + # consistent + xapi_version: "v0.0.0" From 99137b6355342c5c8c116499d32fe39ea41529f1 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 25 Apr 2024 13:27:03 +0100 Subject: [PATCH 21/27] Update README with different build instructions - Removes large package listing from build instructions. - Adds explanation about tags when working from a clone of a fork of the repo. - Includes miscellaneous changes to wording. Signed-off-by: Colin --- README.markdown | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/README.markdown b/README.markdown index a4272105d1b..37174144a3e 100644 --- a/README.markdown +++ b/README.markdown @@ -4,7 +4,7 @@ Xapi Project's XenAPI Management Toolstack ![Build](https://github.com/xapi-project/xen-api/actions/workflows/main.yml/badge.svg?branch=master) Xen API (or xapi) is a management stack that configures and controls -Xen-enabled hosts and resource pools, and co-ordinates resources +Xen-enabled hosts and resource pools, and coordinates resources within the pool. Xapi exposes the Xen API interface for many languages and is a component of the XenServer project. Xen API is written mostly in [OCaml](http://caml.inria.fr/ocaml/) @@ -16,31 +16,31 @@ Xapi is the main component produced by the Linux Foundation's Build and Install ----------------- -To build xen-api from source, we recommend using [opam](https://opam.ocaml.org/doc/Manual.html) with the [xs-opam](https://github.com/xapi-project/xs-opam) repository (farther explanation in its [readme](https://github.com/xapi-project/xs-opam/blob/master/README.md)). +To build xen-api from source, we recommend using [opam](https://opam.ocaml.org/doc/Manual.html) with the [xs-opam](https://github.com/xapi-project/xs-opam) repository (further explanation in its [readme](https://github.com/xapi-project/xs-opam/blob/master/README.md)). 1) Install `opam` and `git` with your package manager. -2) Clone this repo, and work from it's root. +2) Clone this repo and work from its root. ```bash git clone https://github.com/xapi-project/xen-api && cd xen-api ``` -3) Figure out what version of ocaml-base-compiler to use. +3) Figure out which version of ocaml-base-compiler to use. - - Go to [this xs-opam file](https://raw.githubusercontent.com/xapi-project/xs-opam/master/tools/xs-opam-ci.env), and look for "OCAML_VERSION_FULL" - - Run that line, i.e: + - Go to [this xs-opam file](https://raw.githubusercontent.com/xapi-project/xs-opam/master/tools/xs-opam-ci.env), and look for `OCAML_VERSION_FULL`. + - Run that line, e.g.: ```bash - export OCAML_VERSION_FULL="4.10.1" + export OCAML_VERSION_FULL="4.14.1" ``` -4) Setup opam, with your Enviornment (i.e switch). +4) Setup opam with your environment (i.e. switch). ```bash opam init opam switch create xen-api ocaml-base-compiler.$OCAML_VERSION_FULL - # This basically "jumps you into" the enviornment you just created: + # This basically "jumps you into" the environment you just created: eval $(opam env --switch=xen-api --set-switch) ``` @@ -58,12 +58,7 @@ To build xen-api from source, we recommend using [opam](https://opam.ocaml.org/d 6) Install all the Packages. ```bash - PACKAGES="xapi-cli-protocol xapi-client xapi-consts xapi-datamodel xapi-types xapi xe xen-api-sdk xen-api-client xen-api-client-lwt xen-api-client-async xapi-rrdd xapi-rrdd-plugin xapi-rrd-transport xapi-rrd-transport-utils rrd-transport rrdd-plugin rrdd-plugins rrddump gzip http-lib pciutil safe-resources sexpr stunnel uuid xapi-compression xml-light2 zstd vhd-tool xs-toolstack" - - # NOT needed with opam>=2.1.0) Install all the dependencies (Including OS): - opam --yes depext --yes -u $PACKAGES # The first '--yes' is to install depext itself - # Install the Packages finally: - opam install $PACKAGES --yes --deps-only --with-test -v + opam install xs-toolstack # Update the current switch. (You're already on the correct one, just refresh it). eval $(opam env) ``` @@ -78,6 +73,29 @@ To build xen-api from source, we recommend using [opam](https://opam.ocaml.org/d The binaries should now be in `./_build/install/default/bin`! +Working From a Fork +------------------- +If you are working from within a clone of a fork of this repository, you will +need tags from the upstream repository in order to produce a build with the +correct versioning string. + +To fetch these tags, you must ensure that this repository is known to `git` (as a +remote, often called `upstream`) and then you can fetch the tags as follows: + +```bash +git remote add upstream https://github.com/xapi-project/xen-api +git fetch upstream --tags +``` + +You can check if this has been successful by invoking `git describe`. + +You can then push these tags to your remote repository to ensure they are cloned +in future: + +```bash +git push origin --tags +``` + Contributions ------------- From 15391c9f1d86eda4fa5f5d7489ff0256d114627f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Fri, 26 Apr 2024 09:02:45 +0100 Subject: [PATCH 22/27] ci: trim dune cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We should eventually separate the opam dependency install into its own job that uploads an artifact/cache that other jobs can reuse (and that should have a bigger dune cache), however for now just trim the size of the cache. The default is 5GiB, which results in ~1.4GiB compressed, which would hit the 10GiB cache size limit in GitHub way too often. We may need to tweak this value in the future. Signed-off-by: Edwin Török --- .github/workflows/generate-and-build-sdks.yml | 3 +++ .github/workflows/main.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 55ac3f4f09a..ad434d7d9a5 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -42,6 +42,9 @@ jobs: name: SDK_Source_PowerShell path: _build/install/default/xapi/sdk/powershell/* + - name: Trim dune cache + run: opam exec -- dune cache trim --size=2GiB + build-c-sdk: name: Build C SDK runs-on: ubuntu-latest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f2dd84a112..d4bf28aaab2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,3 +49,6 @@ jobs: - name: Check disk space run: df -h || true + + - name: Trim dune cache + run: opam exec -- dune cache trim --size=2GiB From 5dad78b12f502a7285afd0a64e87f50ca626eefe Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Fri, 26 Apr 2024 11:52:31 +0100 Subject: [PATCH 23/27] Removed header because it does not look good on github. Signed-off-by: Konstantina Chremmou --- ocaml/doc/basics.md | 4 ---- ocaml/doc/vm-lifecycle.md | 4 ---- ocaml/doc/wire-protocol.md | 4 ---- ocaml/idl/autogen/api-ref-autogen.md | 4 ---- 4 files changed, 16 deletions(-) diff --git a/ocaml/doc/basics.md b/ocaml/doc/basics.md index b1812da023d..783797051f0 100644 --- a/ocaml/doc/basics.md +++ b/ocaml/doc/basics.md @@ -1,7 +1,3 @@ ---- - layout: doc ---- - # API Basics This document defines the XenServer Management API - an interface for remotely diff --git a/ocaml/doc/vm-lifecycle.md b/ocaml/doc/vm-lifecycle.md index 68664730617..31f31889f36 100644 --- a/ocaml/doc/vm-lifecycle.md +++ b/ocaml/doc/vm-lifecycle.md @@ -1,7 +1,3 @@ ---- - layout: doc ---- - # VM Lifecycle The following diagram shows the states that a VM can be in diff --git a/ocaml/doc/wire-protocol.md b/ocaml/doc/wire-protocol.md index 808d021154d..20e39627cc3 100644 --- a/ocaml/doc/wire-protocol.md +++ b/ocaml/doc/wire-protocol.md @@ -1,7 +1,3 @@ ---- - layout: doc ---- - # Wire Protocol for Remote API Calls API calls are sent over a network to a Xen-enabled host using an RPC protocol. diff --git a/ocaml/idl/autogen/api-ref-autogen.md b/ocaml/idl/autogen/api-ref-autogen.md index 7abf22b6872..1c64c0e8bfb 100644 --- a/ocaml/idl/autogen/api-ref-autogen.md +++ b/ocaml/idl/autogen/api-ref-autogen.md @@ -1,7 +1,3 @@ ---- - layout: doc ---- - # API Reference Version **@xapi-version@** From e87b45943a2e2fc14c313714fc186e6c45f11bea Mon Sep 17 00:00:00 2001 From: Vincent Liu Date: Mon, 29 Apr 2024 10:37:42 +0100 Subject: [PATCH 24/27] Install xapi-tracing-export library xapi-tracing-export is a new library separated from xapi-tracing. This library is needed by xapi-clusterd (and probably other components that require tracing as well). So install it as a library in the Makefile. Signed-off-by: Vincent Liu --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 897862dab52..38c7545cc83 100644 --- a/Makefile +++ b/Makefile @@ -253,7 +253,7 @@ install: build doc sdk doc-json gzip http-lib pciutil sexpr stunnel uuid xml-light2 zstd xapi-compression safe-resources \ message-switch message-switch-async message-switch-cli message-switch-core message-switch-lwt \ message-switch-unix xapi-idl forkexec xapi-forkexecd xapi-storage xapi-storage-script xapi-storage-cli \ - xapi-nbd varstored-guard xapi-log xapi-open-uri xapi-tracing xapi-expiry-alerts cohttp-posix \ + xapi-nbd varstored-guard xapi-log xapi-open-uri xapi-tracing xapi-tracing-export xapi-expiry-alerts cohttp-posix \ xapi-rrd xapi-inventory \ xapi-stdext-date xapi-stdext-encodings xapi-stdext-pervasives xapi-stdext-std xapi-stdext-threads xapi-stdext-unix xapi-stdext-zerocheck # docs @@ -275,7 +275,7 @@ uninstall: gzip http-lib pciutil sexpr stunnel uuid xml-light2 zstd xapi-compression safe-resources \ message-switch message-switch-async message-switch-cli message-switch-core message-switch-lwt \ message-switch-unix xapi-idl forkexec xapi-forkexecd xapi-storage xapi-storage-script xapi-log \ - xapi-open-uri xapi-tracing xapi-expiry-alerts cohttp-posix \ + xapi-open-uri xapi-tracing xapi-tracing-export xapi-expiry-alerts cohttp-posix \ xapi-rrd xapi-inventory \ xapi-stdext-date xapi-stdext-encodings xapi-stdext-pervasives xapi-stdext-std xapi-stdext-threads xapi-stdext-unix xapi-stdext-zerocheck From f442701b71414479a093a4d3979b69a111737703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Mon, 29 Apr 2024 18:09:38 +0100 Subject: [PATCH 25/27] CP-49045: Use thread-safe menhir instead of ocamlyacc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See CA-27687 and CA-30915 and the manual entry https://ocaml.org/manual/5.1/lexyacc.html "Parsers generated by ocamlyacc are not thread-safe. Those parsers rely on an internal work state which is shared by all ocamlyacc generated parsers. The menhir parser generator is a better option if you want thread-safe parsers." We currently hold a global lock while calling the sexpr parser, so even if we try to parse a small sexpression, it'll be blocked behind parsing the large one. Switch to Menhir to make this thread-safe. (We could also consider replacing this sexpression parser with a better maintained one from opam, but there were some peculiarities with how this oneencoded certain values IIRC) Signed-off-by: Edwin Török --- dune-project | 1 + ocaml/libs/sexpr/dune | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dune-project b/dune-project index 20058ef9e72..92d84381b7c 100644 --- a/dune-project +++ b/dune-project @@ -1,5 +1,6 @@ (lang dune 2.0) (formatting (enabled_for ocaml)) +(using menhir 2.0) (generate_opam_files true) diff --git a/ocaml/libs/sexpr/dune b/ocaml/libs/sexpr/dune index 97965164eed..37a60f7cdda 100644 --- a/ocaml/libs/sexpr/dune +++ b/ocaml/libs/sexpr/dune @@ -1,4 +1,4 @@ -(ocamlyacc sExprParser) +(menhir (modules sExprParser)) (ocamllex sExprLexer) From 18dea0ed19d48ae69a3766a195f886127c1c34d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Mon, 29 Apr 2024 18:14:04 +0100 Subject: [PATCH 26/27] CP-49045: drop global lock around sexpression parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TODO: this will need lots of testing and review. Should probably start by reproducing the original issue with a unit test. Signed-off-by: Edwin Török --- ocaml/libs/sexpr/sExpr_TS.ml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ocaml/libs/sexpr/sExpr_TS.ml b/ocaml/libs/sexpr/sExpr_TS.ml index f91fda10413..e68c7d677a8 100644 --- a/ocaml/libs/sexpr/sExpr_TS.ml +++ b/ocaml/libs/sexpr/sExpr_TS.ml @@ -12,11 +12,6 @@ * GNU Lesser General Public License for more details. *) -let lock = Mutex.create () - -let of_string s = - Xapi_stdext_threads.Threadext.Mutex.execute lock (fun () -> - SExprParser.expr SExprLexer.token (Lexing.from_string s) - ) +let of_string s = SExprParser.expr SExprLexer.token (Lexing.from_string s) let string_of = SExpr.string_of From 041f898056f9b58d76f8432374b32dc17f4b1919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Mon, 29 Apr 2024 18:22:05 +0100 Subject: [PATCH 27/27] CP-49045: replace all uses of ocamlyacc with menhir which is thread-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Edwin Török --- ocaml/database/dune | 2 +- ocaml/xenopsd/cli/dune | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ocaml/database/dune b/ocaml/database/dune index 0b0c71425ff..734831adf58 100644 --- a/ocaml/database/dune +++ b/ocaml/database/dune @@ -1,6 +1,6 @@ (ocamllex db_filter_lex) -(ocamlyacc db_filter_parse) +(menhir (modules db_filter_parse)) (library (name xapi_schema) diff --git a/ocaml/xenopsd/cli/dune b/ocaml/xenopsd/cli/dune index 220d5aae2e2..b194b10323c 100644 --- a/ocaml/xenopsd/cli/dune +++ b/ocaml/xenopsd/cli/dune @@ -1,4 +1,4 @@ -(ocamlyacc xn_cfg_parser) +(menhir (modules xn_cfg_parser)) (ocamllex xn_cfg_lexer) (executable