diff --git a/.github/ct.yml b/.github/ct.yml new file mode 100644 index 00000000000..53c9c3d3f17 --- /dev/null +++ b/.github/ct.yml @@ -0,0 +1,20 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 + +# consider helm install to be failed after 10 minutes +helm-extra-args: --timeout 10m +check-version-increment: true +debug: true +chart-dirs: + - deployment/helm +chart-repos: + - stable=https://charts.helm.sh/stable + - bitnami=https://charts.bitnami.com/bitnami diff --git a/.github/kubeval.sh b/.github/kubeval.sh new file mode 100755 index 00000000000..105cdece542 --- /dev/null +++ b/.github/kubeval.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 + +# +# use kubeval to validate helm generated kubernetes manifest +# + +set -o errexit +set -o pipefail + +CHART_DIR=deployment/helm/ditto +KUBEVAL_VERSION="v0.16.1" +SCHEMA_LOCATION="https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/" + +# install kubeval +curl --silent --show-error --fail --location --output /tmp/kubeval.tar.gz https://github.com/instrumenta/kubeval/releases/download/"${KUBEVAL_VERSION}"/kubeval-linux-amd64.tar.gz +sudo tar -C /usr/local/bin -xf /tmp/kubeval.tar.gz kubeval + +# add helm repos to resolve dependencies +helm repo add stable https://charts.helm.sh/stable +helm repo add bitnami https://charts.bitnami.com/bitnami + +# validate chart +echo "helm dependency build..." +helm dependency build "${CHART_DIR}" + +echo "kubeval(idating) ${CHART_DIR} chart..." +helm template "${CHART_DIR}" | kubeval --strict --ignore-missing-schemas --kubernetes-version "${KUBERNETES_VERSION#v}" --schema-location "${SCHEMA_LOCATION}" diff --git a/.github/workflows/docker-nightly.yml b/.github/workflows/docker-nightly.yml index 49163666e37..66633f10181 100644 --- a/.github/workflows/docker-nightly.yml +++ b/.github/workflows/docker-nightly.yml @@ -17,6 +17,7 @@ on: jobs: build: + if: github.repository == 'eclipse-ditto/ditto' runs-on: ubuntu-latest steps: - @@ -129,7 +130,20 @@ jobs: tags: | eclipse/ditto-connectivity:${{ env.IMAGE_TAG }} - - name: Build and push ditto-ui + name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18 + - + name: Install npm dependencies + run: npm install + working-directory: ./ui + - + name: Build UI with node + run: npm run build + working-directory: ./ui + - + name: Build and push ditto-ui image uses: docker/build-push-action@v3 with: context: ./ui diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index d507c2ecf97..0a6aff80868 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -30,6 +30,16 @@ jobs: group: ${{ github.workflow }}-${{ github.ref }} steps: - uses: actions/checkout@v3 + - name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18 + - name: Install npm dependencies + run: npm install + working-directory: ./ui + - name: Build UI with node + run: npm run build + working-directory: ./ui - name: Deploy uses: peaceiris/actions-gh-pages@v3 with: diff --git a/.github/workflows/helm-chart-release.yml b/.github/workflows/helm-chart-release.yml new file mode 100644 index 00000000000..decdbf3f82e --- /dev/null +++ b/.github/workflows/helm-chart-release.yml @@ -0,0 +1,56 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +name: Release and publish Helm chart + +env: + VERSION_HELM: "v3.11.2" +on: + workflow_dispatch: + inputs: + chartVersion: + description: 'Helm chart version' + required: true + type: string + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.VERSION_HELM }} + + - name: Helm | Login + shell: bash + run: echo ${{ secrets.DOCKER_HUB_TOKEN }} | helm registry login -u eclipsedittobot --password-stdin registry-1.docker.io + + - name: Helm | Package + shell: bash + run: helm package deployment/helm/ditto --dependency-update --version ${{ inputs.chartVersion }} + + - name: Helm | Push + shell: bash + run: helm push ditto-${{ inputs.chartVersion }}.tgz oci://registry-1.docker.io/eclipse + + - name: Helm | Logout + shell: bash + run: helm registry logout registry-1.docker.io + + - name: Helm | Output + id: output + shell: bash + run: echo "image=registry-1.docker.io/eclipse/ditto:${{ inputs.chartVersion }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml new file mode 100644 index 00000000000..ef472e9597f --- /dev/null +++ b/.github/workflows/helm-chart.yml @@ -0,0 +1,129 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +name: Lint and test Helm chart + +env: + CONFIG_OPTION_CHART_TESTING: "--config .github/ct.yml" + VERSION_CHART_TESTING: "v3.8.0" + VERSION_HELM: "v3.11.2" + VERSION_PYTHON: "3.9" +on: + pull_request: + paths: + - 'deployment/helm/**' + - '.github/workflows/helm-chart.yml' + - '.github/ct.yml' + - '.github/kubeval.sh' + +jobs: + lint-chart: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.VERSION_HELM }} + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.VERSION_PYTHON }} + check-latest: true + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.4.0 + with: + version: ${{ env.VERSION_CHART_TESTING }} + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed ${{ env.CONFIG_OPTION_CHART_TESTING }} --target-branch ${{ github.event.repository.default_branch }}) + if [[ -n "$changed" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Run chart-testing (lint) + if: steps.list-changed.outputs.changed == 'true' + run: ct lint ${{ env.CONFIG_OPTION_CHART_TESTING }} --target-branch ${{ github.event.repository.default_branch }} + + kubeval-chart: + runs-on: ubuntu-latest + needs: lint-chart + strategy: + matrix: + # the versions supported by kubeval are the ones for + # which a folder exists at + # https://github.com/yannh/kubernetes-json-schema/ + k8s: + - v1.25.2 + - v1.26.4 + - v1.27.1 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Fetch history for chart testing + run: git fetch --prune --unshallow + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.VERSION_HELM }} + - name: Run kubeval + env: + KUBERNETES_VERSION: ${{ matrix.k8s }} + run: .github/kubeval.sh + + install-chart: + name: install-chart + runs-on: ubuntu-latest + needs: + - lint-chart + - kubeval-chart + strategy: + matrix: + # the versions supported by chart-testing are the tags + # available for the docker.io/kindest/node image + # https://hub.docker.com/r/kindest/node/tags + k8s: + - v1.25.2 + - v1.26.4 + - v1.27.1 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Fetch history for chart testing + run: git fetch --prune --unshallow + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.VERSION_HELM }} + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.VERSION_PYTHON }} + check-latest: true + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.4.0 + with: + version: ${{ env.VERSION_CHART_TESTING }} + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed ${{ env.CONFIG_OPTION_CHART_TESTING }} --target-branch ${{ github.event.repository.default_branch }}) + if [[ -n "$changed" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Create kind ${{ matrix.k8s }} cluster + if: steps.list-changed.outputs.changed == 'true' + uses: helm/kind-action@v1.4.0 + with: + node_image: kindest/node:${{ matrix.k8s }} + - name: Run chart-testing (install) + if: steps.list-changed.outputs.changed == 'true' + run: ct install ${{ env.CONFIG_OPTION_CHART_TESTING }} --target-branch ${{ github.event.repository.default_branch }} diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index a6e7224e006..1254d09eb1d 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -29,7 +29,7 @@ jobs: - name: Ensure license year for added files is the file's creation year shell: bash run: | - included_file_endings=".*\.(java|xml|yml)" + included_file_endings=".*\.(java|xml|yml|yaml)" current_year=$(date +'%Y') missing_counter=0 for file in ${{ steps.the-files.outputs.added }}; do diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 716c7e506d5..d51ab5d3516 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -18,6 +18,13 @@ on: # Run build for any PR pull_request: + paths-ignore: + - 'README.md' + - 'RELEASE.md' + - 'CONTRIBUTING.md' + - 'SECURITY.md' + - 'deployment/**' + - 'documentation/**' jobs: build: diff --git a/.github/workflows/push-dockerhub-on-demand.yml b/.github/workflows/push-dockerhub-on-demand.yml index 91c0f382aa1..173394fc299 100644 --- a/.github/workflows/push-dockerhub-on-demand.yml +++ b/.github/workflows/push-dockerhub-on-demand.yml @@ -170,6 +170,19 @@ jobs: eclipse/ditto-connectivity:${{ env.IMAGE_MINOR_TAG }} eclipse/ditto-connectivity:${{ env.IMAGE_MAJOR_TAG }} eclipse/ditto-connectivity:latest + - + name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18 + - + name: Install npm dependencies + run: npm install + working-directory: ./ui + - + name: Build UI with node + run: npm run build + working-directory: ./ui - name: Build and push ditto-ui if: env.MILESTONE_OR_RC_SUFFIX == env.IMAGE_TAG && inputs.dittoImage == 'ditto-ui' diff --git a/.github/workflows/push-dockerhub.yml b/.github/workflows/push-dockerhub.yml index 0b2930c89b7..47189cb50d6 100644 --- a/.github/workflows/push-dockerhub.yml +++ b/.github/workflows/push-dockerhub.yml @@ -155,6 +155,19 @@ jobs: eclipse/ditto-connectivity:${{ env.IMAGE_MINOR_TAG }} eclipse/ditto-connectivity:${{ env.IMAGE_MAJOR_TAG }} eclipse/ditto-connectivity:latest + - + name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18 + - + name: Install npm dependencies + run: npm install + working-directory: ./ui + - + name: Build UI with node + run: npm run build + working-directory: ./ui - name: Build and push ditto-ui if: env.MILESTONE_OR_RC_SUFFIX == env.IMAGE_TAG diff --git a/.gitignore b/.gitignore index e28468ceb25..bb0a6751374 100755 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,5 @@ deployment/helm/eclipse-ditto/requirements.lock deployment/helm/eclipse-ditto/charts/* .factorypath ui/node_modules -ui/package.json -ui/package-lock.json +ui/dist diff --git a/.run/ConnectivityService.run.xml b/.run/ConnectivityService.run.xml index fd0fa2ff98d..19697e5c888 100644 --- a/.run/ConnectivityService.run.xml +++ b/.run/ConnectivityService.run.xml @@ -1,31 +1,15 @@ - + + - + \ No newline at end of file diff --git a/.run/Ditto.run.xml b/.run/Ditto.run.xml index ac5bcbb86b2..6aacf9255db 100644 --- a/.run/Ditto.run.xml +++ b/.run/Ditto.run.xml @@ -1,15 +1,3 @@ - diff --git a/.run/GatewayService.run.xml b/.run/GatewayService.run.xml index 7884b94f5bc..de83356697b 100644 --- a/.run/GatewayService.run.xml +++ b/.run/GatewayService.run.xml @@ -1,15 +1,3 @@ - @@ -18,12 +6,6 @@ \ No newline at end of file diff --git a/.run/SearchService.run.xml b/.run/SearchService.run.xml index 98ee0fb7b41..25444065eaa 100644 --- a/.run/SearchService.run.xml +++ b/.run/SearchService.run.xml @@ -1,15 +1,3 @@ - @@ -23,12 +11,6 @@ \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1cb0974a9f3..09328574e5c 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,7 @@ Please make sure any file you newly create contains a proper license header. Fin Adjusted for Java classes: ```java /* - * Copyright (c) 2019 Contributors to the Eclipse Foundation + * Copyright (c) 2023 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -72,7 +72,7 @@ Adjusted for Java classes: Adjusted for XML files: ```xml ``` -### Important - -Please do not forget to add your name/organization to the [legal/NOTICE.md](legal/NOTICE.md) file's Copyright Holders -section. If this is not the first contribution you make, then simply update the time period contained in the copyright -entry to use the year of your first contribution as the lower boundary and the current year as the upper boundary, e.g. - - Copyright 2018-2019 ACME Corporation - ## Submitting the Changes Submit a pull request via the normal GitHub UI. @@ -100,3 +92,128 @@ Submit a pull request via the normal GitHub UI. ## After Submitting * Do not use your branch for any other development, otherwise further changes that you make will be visible in the PR. + + +# OSS development process - rules + +As of 02/2023, the following additional "rules" regarding the open OSS development process were agreed on. + +## Addition of new features via feature toggles + +Goals: +* Reduce the risk for other Ditto users that a new feature have an impact on existing functionality and stability +* Whenever possible (and feasible), added functionality shall be added using a "feature toggle". + +Ditto already has a class `FeatureToggle.java` where feature toggles are contained and providing functionality to +"secure" a feature with a method in there which throws an `UnsupportedSignalException` once a feature is used which +is disabled via feature toggle. + +The toggles are then configured in `ditto-devops.conf` file and can be enabled/disabled via the contained environment variables. + +## Creating GitHub issues before starting to work on code + +Goals: +* Improve transparency on what is currently happening +* Openly discuss new features and whether they are a good fit for Ditto +* Reduce "time waste" + +Whenever a new feature or a bugfix is being worked on, we want to create an issue in Ditto's GitHub project **beforehand**: +https://github.com/eclipse-ditto/ditto/issues + +This provides the needed transparency for other contributors before much effort is put into a new topic in order to: +* Get input on the background (e.g. use case behind / "the need") of the feature/bugfix +* Provide feedback, maybe even suggesting alternatives instead +* Provide suggestions of how to implement it the most efficient way +* Maybe even find synergies when more than 1 contributing companies currently have the same topic to work on + +The following situation shall be prevented: +* If no issue is created upfront, a contributing company e.g. invests 2 months of work in a new feature +* Then a PR is created with this new functionality +* Only then, a discussion with other contributors can start +* At this point, when there e.g. is a big flaw in the architecture or security or API stability of the added functionality, + the invested 2 months could - in the worst case - be a complete waste of time +* This could easily be resolved by discussing it beforehand + +## Create PullRequests early + +Goals: +* Get early feedback on implementation details of new features / bugfixes +* Prevent that an implementation goes "into the wrong direction" (e.g. performance or security wise) + +PullRequests should be created quite early and publicly on the Ditto project. +If they are not yet "ready" to review/merge, they must be marked as "DRAFT" - once they are ready, they can be marked +as such and a review can be performed. + +## Make use of GitHub "Projects" for showing current work for next release + +Goals: +* Make transparent "who" currently works on "what" +* Make transparent what the current agenda for the next Ditto release is + +The new "Projects" capabilities of GitHub look more than sufficient of what we want to achieve here: +* https://github.com/orgs/eclipse-ditto/projects/1 +* https://github.com/orgs/eclipse-ditto/projects/1/views/2 (table view is especially useful, as grouping by "Milestone" is necessary) + +## Establish system-tests in the OpenSource codebase + +Goals: +* Provide means to run automated tests for future enhancements to Ditto +* Secure existing functionality, avoid breaking APIs and existing functionality when changes to the Ditto OSS codebase are done + +The system tests for Eclipse Ditto were initiated here: +https://github.com/eclipse-ditto/ditto-testing + +The tests should be part of the validations done in a PR before a PR is approved and merged. +In order to be able to do that, we want to clarify if the Eclipse Foundation can provide enough resources in order to +run the system-tests in a stable way. + +Currently, that seems to be quite difficult, as projects only have very limited resources in order to run their builds. +In addition, the CI runs in an OpenShift cluster with additional restrictions, e.g. regarding the kind of Docker images +which can be run, exposing of the Docker socket, etc. + +## Regular community meetings + +Goals: + +* Discuss upcoming topics/problems in advance +* Stay in touch via audio/video +* Build up a (contributor and adopter) community who can help each other + +We want to re-establish regular community meetings/call, e.g. a meeting every 2 weeks for 1 hour. +We can utilize the Zoom account from the Eclipse Foundation to have a "neutral" one .. or just use "Google Meet". + +## Chat for (internal) exchanges + +Goals: +* Have a direct channel where to reach other Ditto committers and contributors +* In order to get timely responses if e.g. a bugfix release has to be scheduled/done quickly + +We can use "Gitter.IM" communities to add different rooms of which some also can be private: +https://gitter.im/EclipseDitto/community + +## Release strategy + +Goals: +* Have rules of how often to do "planned feature releases" +* Have options for contributing companies to prioritize a release (e.g. if urgent bugfix or urgent feature release) + +The suggestion would be to have approximately 4 planned minor releases per year, 1 each quarter (e.g. 03 / 06 / 09 / 12). +If needed and all contributing companies agree minor releases can also happen earlier/more often. + +Bugfix releases should be done immediately if a critical bug was fixed and either the contributors or the community need a quick fix release. + +## Approving / merging PRs + +Goals: +* PullRequests - once they are ready - shall not stay unmerged for a long time as this leads to the risk they are not + mergable or get outdated quickly + +Approach: + +* Before merging a PR at least 1 approval is required + * Approvals shall only be issued after a code review + * Preferably that would be 1 approval from an existing Ditto committer + * But could also be the approval of an active contributor who does not yet have committer status +* If no approval is given for a PR within a duration of 4 weeks after declaring it "ready", a PR can also be merged without other approvals + * Before doing so, the reasons for not approving must be found out (e.g. via the Chat / community call) + * If the reason simply is "no time" and there are no objections against that PR, the PR can be merged without other approvals diff --git a/NOTICE.md b/NOTICE.md index 3e022ace3d9..d53c4ca9742 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1,6 +1,6 @@ This content is produced and maintained by the Eclipse Ditto project. -* Project home: https://www.eclipse.org/ditto +* Project home: https://www.eclipse.dev/ditto # Trademarks @@ -23,7 +23,7 @@ SPDX-License-Identifier: EPL-2.0 # Source Code -* https://github.com/eclipse/ditto +* https://github.com/eclipse-ditto/ditto # Third-party Content diff --git a/README.md b/README.md index 71564ff8b25..b5f5169f07d 100755 --- a/README.md +++ b/README.md @@ -7,20 +7,20 @@ # Eclipse Ditto™ [![Join the chat at https://gitter.im/eclipse/ditto](https://badges.gitter.im/eclipse/ditto.svg)](https://gitter.im/eclipse/ditto?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Build Status](https://github.com/eclipse/ditto/workflows/build/badge.svg)](https://github.com/eclipse/ditto/actions?query=workflow%3Abuild) +[![Build Status](https://github.com/eclipse-ditto/ditto/workflows/build/badge.svg)](https://github.com/eclipse-ditto/ditto/actions?query=workflow%3Abuild) [![Maven Central](https://img.shields.io/maven-central/v/org.eclipse.ditto/ditto?label=maven)](https://search.maven.org/search?q=g:org.eclipse.ditto) [![Docker pulls](https://img.shields.io/docker/pulls/eclipse/ditto-things.svg)](https://hub.docker.com/search?q=eclipse%2Fditto&type=image) [![License](https://img.shields.io/badge/License-EPL%202.0-green.svg)](https://opensource.org/licenses/EPL-2.0) [![Lines of code](https://img.shields.io/badge/dynamic/xml.svg?label=Lines%20of%20code&url=https%3A%2F%2Fwww.openhub.net%2Fprojects%2Feclipse-ditto.xml%3Fapi_key%3D11ac3aa12a364fd87b461559a7eedcc53e18fb5a4cf1e43e02cb7a615f1f3d4f&query=%2Fresponse%2Fresult%2Fproject%2Fanalysis%2Ftotal_code_lines&colorB=lightgrey)](https://www.openhub.net/p/eclipse-ditto) -[Eclipse Ditto](https://www.eclipse.org/ditto/)™ is a technology in the IoT implementing a software pattern called “digital twins”. +[Eclipse Ditto](https://www.eclipse.dev/ditto/)™ is a technology in the IoT implementing a software pattern called “digital twins”. A digital twin is a virtual, cloud based, representation of his real world counterpart (real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations, …). An ever growing list of [adopters](https://iot.eclipse.org/adopters/?#iot.ditto) makes use of Ditto as part of their IoT platforms - if you're as well using it, it would be super nice to show your [adoption here](https://iot.eclipse.org/adopters/how-to-be-listed-as-an-adopter/). ## Documentation -Find the documentation on the project site: [https://www.eclipse.org/ditto/](https://www.eclipse.org/ditto/) +Find the documentation on the project site: [https://www.eclipse.dev/ditto/](https://www.eclipse.dev/ditto/) ## Eclipse Ditto™ explorer UI @@ -30,7 +30,7 @@ You should be able to work with your locally running default using the `local_di ## Star History -[![Star History Chart](https://api.star-history.com/svg?repos=eclipse-ditto/ditto&type=Date)](https://star-history.com/?secret=Z2hwXzJERUNBUmFRa09KM3BvdTFMUkJ1Y3VnY25FV3hxVjNBM3hEVQ==#eclipse-ditto/ditto&Date) +[![Star History Chart](https://api.star-history.com/svg?repos=eclipse-ditto/ditto&type=Date)](https://star-history.com/#eclipse-ditto/ditto&Date) ## Getting started @@ -41,7 +41,7 @@ In order to start up Ditto via *Docker Compose*, you'll need: * at least 2 CPU cores which can be used by Docker * at least 4 GB of RAM which can be used by Docker -You also have other possibilities to run Ditto, please have a look [here](https://github.com/eclipse/ditto/tree/master/deployment) to explore them. +You also have other possibilities to run Ditto, please have a look [here](https://github.com/eclipse-ditto/ditto/tree/master/deployment) to explore them. ### Start Ditto @@ -58,7 +58,7 @@ docker-compose logs -f ``` Open following URL to get started: [http://localhost:8080](http://localhost:8080)
-Or have a look at the ["Hello World"](https://www.eclipse.org/ditto/intro-hello-world.html) +Or have a look at the ["Hello World"](https://www.eclipse.dev/ditto/intro-hello-world.html) Additional [deployment options](deployment/) are also available, if Docker Compose is not what you want to use. diff --git a/RELEASE.md b/RELEASE.md index b986c398bd4..cc93d5667b0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -14,21 +14,21 @@ Ditto releases are tracked and planned here: https://projects.eclipse.org/projec * First close the staging repo (after all artifacts are there, e.g. also the client artifacts) * Then release the staging repo * Then it will take a few hours until those changes are synced successfully to Maven central -* Write Release notes, e.g. like for 3.1.0: https://www.eclipse.org/ditto/release_notes_310.html +* Write Release notes, e.g. like for 3.1.0: https://www.eclipse.dev/ditto/release_notes_310.html * New features, changes, bug fixes to last release / milestone release * Add migration notes (if there are any) -* Write a Blog post announcement, e.g. like for: https://www.eclipse.org/ditto/2022-12-16-release-announcement-310.html -* Close GitHub milestone (and assign all Issues/PRs which were still included in that milestone): https://github.com/eclipse/ditto/milestones -* Create a GitHub release: https://github.com/eclipse/ditto/releases (based on the Tags which was pushed during release job) +* Write a Blog post announcement, e.g. like for: https://www.eclipse.dev/ditto/2022-12-16-release-announcement-310.html +* Close GitHub milestone (and assign all Issues/PRs which were still included in that milestone): https://github.com/eclipse-ditto/ditto/milestones +* Create a GitHub release: https://github.com/eclipse-ditto/ditto/releases (based on the Tags which was pushed during release job) * Write a mail to the "ditto-dev" mailing list * Tweet about it ;) * Set binary compatibility check version to the new public release. Delete all exclusions and module-level deactivation of japi-cmp plugin except for *.internal packages. -* Update https://github.com/eclipse/ditto/blob/master/SECURITY.md with the supported versions to receive security fixes +* Update https://github.com/eclipse-ditto/ditto/blob/master/SECURITY.md with the supported versions to receive security fixes * For major+minor versions: * Create a "release" branch"release-" from the released git tag * needed to build the documentation from * required for bugfixes to build a bugfix release for the affected minor version - * Add the new version to the documentation config: https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/_config.yml#L114 + * Add the new version to the documentation config: https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/_config.yml#L114 * Adjust the "website" CI jobs to also build the newly added branch: * https://ci.eclipse.org/ditto/view/Website/ * https://ci.eclipse.org/ditto/view/Website/job/website-build-and-deploy-fast/ will build the latest released minor version + "master" (development) version diff --git a/SECURITY.md b/SECURITY.md index db6acad65fa..14caa8332fe 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,14 +1,17 @@ # Security Policy +Eclipse Ditto follows the [Eclipse Vulnerability Reporting Policy](https://www.eclipse.org/security/policy.php). Vulnerabilities are tracked by the Eclipse security team, in cooperation with the Ditto project leads. Fixing vulnerabilities is taken care of by the Ditto project committers, with assistance and guidance of the security team. + ## Supported Versions +Eclipse Ditto provides security updates for the two most recent minor versions. These versions of Eclipse Ditto are currently being supported with security updates. | Version | Supported | |---------| ------------------ | -| 3.1.x | :white_check_mark: | -| 3.0.x | :white_check_mark: | -| < 3.0.0 | :x: | +| 3.3.x | :white_check_mark: | +| 3.2.x | :white_check_mark: | +| < 3.2.0 | :x: | ## Reporting a Vulnerability diff --git a/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceResponseTest.java b/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceResponseTest.java index b46a3584672..3e5406a7965 100644 --- a/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceResponseTest.java +++ b/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceResponseTest.java @@ -20,6 +20,7 @@ import org.eclipse.ditto.base.model.common.HttpStatus; import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.entity.type.EntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; @@ -33,7 +34,7 @@ */ public final class CleanupPersistenceResponseTest { - private static final EntityId ID = EntityId.of(EntityType.of("thing"), "eclipse:ditto"); + private static final EntityId ID = NamespacedEntityId.of(EntityType.of("thing"), "eclipse:ditto"); private static final JsonObject KNOWN_JSON = JsonObject.newBuilder() .set(CommandResponse.JsonFields.TYPE, CleanupPersistenceResponse.TYPE) .set(CleanupCommandResponse.JsonFields.ENTITY_TYPE, ID.getEntityType().toString()) diff --git a/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceTest.java b/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceTest.java index c1de951f4bd..43ba4297412 100644 --- a/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceTest.java +++ b/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceTest.java @@ -18,6 +18,7 @@ import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.entity.type.EntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.signals.commands.Command; @@ -31,7 +32,7 @@ */ public class CleanupPersistenceTest { - private static final EntityId ID = EntityId.of(EntityType.of("thing"), "eclipse:ditto"); + private static final EntityId ID = NamespacedEntityId.of(EntityType.of("thing"), "eclipse:ditto"); private static final JsonObject KNOWN_JSON = JsonObject.newBuilder() .set(Command.JsonFields.TYPE, CleanupPersistence.TYPE) .set(CleanupCommand.JsonFields.ENTITY_TYPE, ID.getEntityType().toString()) diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/acks/AcknowledgementLabelInvalidException.java b/base/model/src/main/java/org/eclipse/ditto/base/model/acks/AcknowledgementLabelInvalidException.java index 8dbc94eed48..03f40f04390 100644 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/acks/AcknowledgementLabelInvalidException.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/acks/AcknowledgementLabelInvalidException.java @@ -19,12 +19,12 @@ import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.NotThreadSafe; -import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.base.model.common.HttpStatus; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; /** * Thrown if an AcknowledgementLabel is not valid, for example because it did not comply to the AcknowledgmentLabel @@ -48,7 +48,7 @@ public final class AcknowledgementLabelInvalidException extends DittoRuntimeExce "An acknowledgement label must conform to the regular expression of Ditto documentation."; private static final URI DEFAULT_HREF = URI.create( - "https://www.eclipse.org/ditto/protocol-specification-topic.html#acknowledgement-criterion-actions"); + "https://www.eclipse.dev/ditto/protocol-specification-topic.html#acknowledgement-criterion-actions"); private static final long serialVersionUID = -2385649293006205966L; diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/common/LikeHelper.java b/base/model/src/main/java/org/eclipse/ditto/base/model/common/LikeHelper.java index 75a3d9bd566..e25db8a962b 100644 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/common/LikeHelper.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/common/LikeHelper.java @@ -14,6 +14,8 @@ import java.util.regex.Pattern; +import javax.annotation.Nullable; + /** * A helper to create "like" patterns. * @@ -40,7 +42,8 @@ private LikeHelper() { * @param expression The wildcard expression to convert. * @return The regular expression, which can be compiled with {@link Pattern#compile(String)}. */ - public static String convertToRegexSyntax(final String expression) { + @Nullable + public static String convertToRegexSyntax(@Nullable final String expression) { if (expression == null) { return null; } diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/EntityId.java b/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/EntityId.java index 32566d5fd02..47f71909a7a 100644 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/EntityId.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/EntityId.java @@ -35,7 +35,13 @@ public interface EntityId extends CharSequence, Comparable { */ static EntityId of(final EntityType entityType, final CharSequence entityId) { final EntityIds entityIds = EntityIds.getInstance(); - return entityIds.getEntityId(entityType, entityId); + try { + // most entity ids are namespaces, so try that first + return entityIds.getNamespacedEntityId(entityType, entityId); + } catch (final NamespacedEntityIdInvalidException namespacedEntityIdInvalidException) { + // only in the exceptional case, fall back to non-namespaced flavor: + return entityIds.getEntityId(entityType, entityId); + } } @Override diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/NamespacedEntityIdInvalidException.java b/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/NamespacedEntityIdInvalidException.java index 4843ff97940..da4d16a030a 100644 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/NamespacedEntityIdInvalidException.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/NamespacedEntityIdInvalidException.java @@ -56,7 +56,7 @@ public final class NamespacedEntityIdInvalidException extends EntityIdInvalidExc "length of 256 characters."; private static final URI DEFAULT_HREF = - URI.create("https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id"); + URI.create("https://www.eclipse.dev/ditto/basic-namespaces-and-names.html#namespaced-id"); private static final long serialVersionUID = -8903476318490123234L; diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/AbstractDittoHeaders.java b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/AbstractDittoHeaders.java index aaa6b176a0e..f7c32d086fe 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/AbstractDittoHeaders.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/AbstractDittoHeaders.java @@ -313,6 +313,12 @@ public Optional getIfNoneMatch() { .map(EntityTagMatchers::fromCommaSeparatedString); } + @Override + public Optional getIfEqual() { + return getStringForDefinition(DittoHeaderDefinition.IF_EQUAL) + .flatMap(IfEqual::forOption); + } + @Override public Optional getInboundPayloadMapper() { return getStringForDefinition(DittoHeaderDefinition.INBOUND_PAYLOAD_MAPPER); diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/AbstractDittoHeadersBuilder.java b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/AbstractDittoHeadersBuilder.java index 3dde80d8717..f91f8737923 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/AbstractDittoHeadersBuilder.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/AbstractDittoHeadersBuilder.java @@ -410,6 +410,12 @@ public S ifNoneMatch(final EntityTagMatchers entityTags) { return myself; } + @Override + public S ifEqual(final IfEqual ifEqual) { + putCharSequence(DittoHeaderDefinition.IF_EQUAL, ifEqual.toString()); + return myself; + } + @Override public S inboundPayloadMapper(@Nullable final String inboundPayloadMapperId) { putCharSequence(DittoHeaderDefinition.INBOUND_PAYLOAD_MAPPER, inboundPayloadMapperId); diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java index b8f520c80a4..438418c524a 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java @@ -195,6 +195,21 @@ public enum DittoHeaderDefinition implements HeaderDefinition { false, HeaderValueValidators.getEntityTagMatchersValidator()), + /** + * Header definition for "If-Equal". + * Can hold one of the values: {@code update}, {@code skip}. + *

+ * Key: {@code "If-Equal"}, Java type: {@link String}. + *

+ * @since 3.3.0 + */ + IF_EQUAL("if-equal", + IfEqual.class, + String.class, + true, + false, + HeaderValueValidators.getEnumValidator(IfEqual.values())), + /** * Header definition for the internal header "ditto-reply-target". This header is evaluated for responses to be * published. @@ -349,7 +364,7 @@ public enum DittoHeaderDefinition implements HeaderDefinition { * * @since 3.0.0 */ - DITTO_METADATA("ditto-metadata", JsonObject.class, false, true, HeaderValueValidators.getNoOpValidator()), + DITTO_METADATA("ditto-metadata", JsonObject.class, false, true, HeaderValueValidators.getJsonObjectValidator()), /** * Header definition for allowing the policy lockout (i.e. a subject can create a policy without having WRITE @@ -484,7 +499,43 @@ public enum DittoHeaderDefinition implements HeaderDefinition { Boolean.class, false, true, - HeaderValueValidators.getBooleanValidator()); + HeaderValueValidators.getBooleanValidator()), + + /** + * Header containing a specific historical revision to retrieve when retrieving a persisted entity + * (thing/policy/connection). + * + * @since 3.2.0 + */ + AT_HISTORICAL_REVISION("at-historical-revision", + Long.class, + true, + false, + HeaderValueValidators.getLongValidator()), + + /** + * Header containing a specific historical timestamp to retrieve when retrieving a persisted entity + * (thing/policy/connection). + * + * @since 3.2.0 + */ + AT_HISTORICAL_TIMESTAMP("at-historical-timestamp", + String.class, + true, + false, + HeaderValueValidators.getNoOpValidator()), + + /** + * Header containing retrieved historical headers to be returned for e.g. a historical retrieve command. + * Useful for audit-log information, e.g. which "originator" did a change to a thing/policy/connection. + * + * @since 3.2.0 + */ + HISTORICAL_HEADERS("historical-headers", + JsonObject.class, + false, + true, + HeaderValueValidators.getJsonObjectValidator()); /** * Map to speed up lookup of header definition by key. diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaders.java b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaders.java index 0e48413c85d..5bc4497cfad 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaders.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaders.java @@ -266,6 +266,14 @@ static DittoHeadersBuilder newBuilder(final JsonObject jsonObject) { */ Optional getIfNoneMatch(); + /** + * Returns the "If-Equal" header defining whether to update a value if it was equal to the previous value or not. + * + * @return the if-equal header. + * @since 3.3.0 + */ + Optional getIfEqual(); + /** * Returns the inbound {@code MessageMapper} ID which mapped incoming arbitrary payload from external sources. * diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeadersBuilder.java b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeadersBuilder.java index 1ded962e980..6fac2c3c2c3 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeadersBuilder.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeadersBuilder.java @@ -189,6 +189,16 @@ default B randomCorrelationId() { */ B ifNoneMatch(EntityTagMatchers entityTags); + /** + * Sets the If-Equal value. + * + * @param ifEqual The if-equal value to set defining what to do with a value to update which is equal to + * its previous value. + * @return this builder for Method Chaining + * @since 3.3.0 + */ + B ifEqual(IfEqual ifEqual); + /** * Sets the inbound {@code MessageMapper} ID value. * diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/IfEqual.java b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/IfEqual.java new file mode 100644 index 00000000000..90e833a8a59 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/IfEqual.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.headers; + +import java.util.Arrays; +import java.util.Optional; + +/** + * Possible options for Ditto's {@code if-equal} header. + * + * @since 3.3.0 + */ +public enum IfEqual { + + /** + * Option which updates a value, even if the value is the same (via {@code equal()}) than the value before. + * This is the default if omitted and for backwards compatibility. + */ + UPDATE("update"), + + /** + * Option which will skip the update of a twin if the new value is the same (via {@code equal()}) than the value + * before. + */ + SKIP("skip"); + + private final String option; + + IfEqual(final String option) { + this.option = option; + } + + @Override + public String toString() { + return option; + } + + /** + * Find an If-Equal option by a provided option string. + * + * @param option the option. + * @return the option with the given option string if any exists. + */ + public static Optional forOption(final String option) { + return Arrays.stream(values()) + .filter(strategy -> strategy.toString().equals(option)) + .findAny(); + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java index 07ee3bcc1d8..7bfc5bb5c9c 100644 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java @@ -33,6 +33,12 @@ public final class FeatureToggle { */ public static final String WOT_INTEGRATION_ENABLED = "ditto.devops.feature.wot-integration-enabled"; + /** + * System property name of the property defining whether the historical API access is enabled. + * @since 3.2.0 + */ + public static final String HISTORICAL_APIS_ENABLED = "ditto.devops.feature.historical-apis-enabled"; + /** * Resolves the system property {@value MERGE_THINGS_ENABLED}. */ @@ -43,6 +49,11 @@ public final class FeatureToggle { */ private static final boolean IS_WOT_INTEGRATION_ENABLED = resolveProperty(WOT_INTEGRATION_ENABLED); + /** + * Resolves the system property {@value HISTORICAL_APIS_ENABLED}. + */ + private static final boolean IS_HISTORICAL_APIS_ENABLED = resolveProperty(HISTORICAL_APIS_ENABLED); + private static boolean resolveProperty(final String propertyName) { final String propertyValue = System.getProperty(propertyName, Boolean.TRUE.toString()); return !Boolean.FALSE.toString().equalsIgnoreCase(propertyValue); @@ -101,4 +112,35 @@ public static DittoHeaders checkWotIntegrationFeatureEnabled(final String signal public static boolean isWotIntegrationFeatureEnabled() { return IS_WOT_INTEGRATION_ENABLED; } + + /** + * Checks if the historical API access feature is enabled based on the system property {@value HISTORICAL_APIS_ENABLED}. + * + * @param signal the name of the signal that was supposed to be processed + * @param dittoHeaders headers used to build exception + * @return the unmodified headers parameters + * @throws UnsupportedSignalException if the system property + * {@value HISTORICAL_APIS_ENABLED} resolves to {@code false} + * @since 3.2.0 + */ + public static DittoHeaders checkHistoricalApiAccessFeatureEnabled(final String signal, final DittoHeaders dittoHeaders) { + if (!isHistoricalApiAccessFeatureEnabled()) { + throw UnsupportedSignalException + .newBuilder(signal) + .dittoHeaders(dittoHeaders) + .build(); + } + return dittoHeaders; + } + + /** + * Returns whether the historical API access feature is enabled based on the system property + * {@value HISTORICAL_APIS_ENABLED}. + * + * @return whether the historical API access feature is enabled or not. + * @since 3.2.0 + */ + public static boolean isHistoricalApiAccessFeatureEnabled() { + return IS_HISTORICAL_APIS_ENABLED; + } } diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/WithStreamingSubscriptionId.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/WithStreamingSubscriptionId.java new file mode 100755 index 00000000000..a6810d5113a --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/WithStreamingSubscriptionId.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals; + +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; + +/** + * Interface of streaming commands/events addressing a particular session identified by a subscription ID. + * + * @since 3.2.0 + */ +public interface WithStreamingSubscriptionId> extends + StreamingSubscriptionCommand { + + /** + * Returns the subscriptionId identifying the session of this streaming signal. + * + * @return the subscriptionId. + */ + String getSubscriptionId(); + + /** + * Json fields of this command. + */ + final class JsonFields { + + /** + * JSON field for the streaming subscription ID. + */ + public static final JsonFieldDefinition SUBSCRIPTION_ID = + JsonFactory.newStringFieldDefinition("subscriptionId"); + + JsonFields() { + throw new AssertionError(); + } + + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/Command.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/Command.java index 4afd8b0ff4b..c4277fa6960 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/Command.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/Command.java @@ -159,7 +159,13 @@ enum Category { * Category of commands that are neither of the above 3 (query, modify, delete) but perform an action on the * entity. */ - ACTION; + ACTION, + + /** + * Category of commands that stream e.g. historical events. + * @since 3.2.0 + */ + STREAM; /** * Determines whether the passed {@code category} effectively modifies the targeted entity. diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionNotFoundException.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionNotFoundException.java new file mode 100755 index 00000000000..f0a4c9e3e03 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionNotFoundException.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.exceptions; + +import java.net.URI; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.exceptions.GeneralException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Error response for streaming subscription commands addressing a nonexistent subscription. + * + * @since 3.2.0 + */ +@JsonParsableException(errorCode = StreamingSubscriptionNotFoundException.ERROR_CODE) +public class StreamingSubscriptionNotFoundException extends DittoRuntimeException implements GeneralException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "streaming.subscription.not.found"; + + private static final HttpStatus STATUS_CODE = HttpStatus.NOT_FOUND; + + private StreamingSubscriptionNotFoundException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + + super(ERROR_CODE, STATUS_CODE, dittoHeaders, message, description, cause, href); + } + + /** + * Create a {@code StreamingSubscriptionNotFoundException}. + * + * @param subscriptionId ID of the nonexistent subscription. + * @param dittoHeaders the Ditto headers. + * @return the exception. + */ + public static StreamingSubscriptionNotFoundException of(final String subscriptionId, final DittoHeaders dittoHeaders) { + return new Builder() + .message(String.format("No subscription with ID '%s' exists.", subscriptionId)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * Constructs a new {@code StreamingSubscriptionNotFoundException} object with the exception message extracted from the + * given JSON object. + * + * @param jsonObject the JSON to read the {@link DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new StreamingSubscriptionNotFoundException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionNotFoundException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link StreamingSubscriptionNotFoundException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() {} + + @Override + protected StreamingSubscriptionNotFoundException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new StreamingSubscriptionNotFoundException(dittoHeaders, message, description, cause, href); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionProtocolErrorException.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionProtocolErrorException.java new file mode 100755 index 00000000000..5875e8660db --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionProtocolErrorException.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.exceptions; + +import java.net.URI; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.exceptions.GeneralException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Error response for subscriptions with no interaction for a long time. + * + * @since 3.2.0 + */ +@JsonParsableException(errorCode = StreamingSubscriptionProtocolErrorException.ERROR_CODE) +public class StreamingSubscriptionProtocolErrorException extends DittoRuntimeException implements GeneralException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "streaming.subscription.protocol.error"; + + private static final HttpStatus STATUS_CODE = HttpStatus.BAD_REQUEST; + + private StreamingSubscriptionProtocolErrorException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + + super(ERROR_CODE, STATUS_CODE, dittoHeaders, message, description, cause, href); + } + + /** + * Create a {@code StreamingSubscriptionProtocolErrorException}. + * + * @param cause the actual protocol error. + * @param dittoHeaders the Ditto headers. + * @return the exception. + */ + public static StreamingSubscriptionProtocolErrorException of(final Throwable cause, final DittoHeaders dittoHeaders) { + return new Builder() + .message(cause.getMessage()) + .cause(cause) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * Create an empty builder for this exception. + * + * @return an empty builder. + */ + public static DittoRuntimeExceptionBuilder newBuilder() { + return new Builder(); + } + + /** + * Constructs a new {@code StreamingSubscriptionProtocolErrorException} object with the exception message extracted from the + * given JSON object. + * + * @param jsonObject the JSON to read the {@link DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new StreamingSubscriptionProtocolErrorException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionProtocolErrorException fromJson(final JsonObject jsonObject, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link StreamingSubscriptionProtocolErrorException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() {} + + @Override + protected StreamingSubscriptionProtocolErrorException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new StreamingSubscriptionProtocolErrorException(dittoHeaders, message, description, cause, href); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionTimeoutException.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionTimeoutException.java new file mode 100755 index 00000000000..a55feff9a31 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionTimeoutException.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.exceptions; + +import java.net.URI; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.exceptions.GeneralException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Error response for subscriptions with no interaction for a long time. + * + * @since 3.2.0 + */ +@JsonParsableException(errorCode = StreamingSubscriptionTimeoutException.ERROR_CODE) +public class StreamingSubscriptionTimeoutException extends DittoRuntimeException implements GeneralException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "streaming.subscription.timeout"; + + private static final HttpStatus STATUS_CODE = HttpStatus.REQUEST_TIMEOUT; + + private StreamingSubscriptionTimeoutException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + + super(ERROR_CODE, STATUS_CODE, dittoHeaders, message, description, cause, href); + } + + /** + * Create a {@code StreamingSubscriptionTimeoutException}. + * + * @param subscriptionId ID of the nonexistent subscription. + * @param dittoHeaders the Ditto headers. + * @return the exception. + */ + public static StreamingSubscriptionTimeoutException of(final String subscriptionId, final DittoHeaders dittoHeaders) { + return new Builder() + .message(String.format("The subscription '%s' stopped due to a lack of interaction.", subscriptionId)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * Constructs a new {@code StreamingSubscriptionTimeoutException} object with the exception message extracted from the + * given JSON object. + * + * @param jsonObject the JSON to read the {@link DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new StreamingSubscriptionTimeoutException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionTimeoutException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link StreamingSubscriptionTimeoutException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() {} + + @Override + protected StreamingSubscriptionTimeoutException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new StreamingSubscriptionTimeoutException(dittoHeaders, message, description, cause, href); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/AbstractStreamingSubscriptionCommand.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/AbstractStreamingSubscriptionCommand.java new file mode 100755 index 00000000000..1b8ff670875 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/AbstractStreamingSubscriptionCommand.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.util.Objects; +import java.util.function.Predicate; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.EntityIdJsonDeserializer; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.entity.type.EntityTypeJsonDeserializer; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.commands.AbstractCommand; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * Abstract base class for streaming commands. + * + * @param the type of the AbstractStreamingSubscriptionCommand + * @since 3.2.0 + */ +@Immutable +abstract class AbstractStreamingSubscriptionCommand> + extends AbstractCommand + implements StreamingSubscriptionCommand { + + protected final EntityId entityId; + protected final JsonPointer resourcePath; + + protected AbstractStreamingSubscriptionCommand(final String type, + final EntityId entityId, + final JsonPointer resourcePath, + final DittoHeaders dittoHeaders) { + + super(type, dittoHeaders); + this.entityId = checkNotNull(entityId, "entityId"); + this.resourcePath = checkNotNull(resourcePath, "resourcePath"); + } + + protected static EntityId deserializeEntityId(final JsonObject jsonObject) { + return EntityIdJsonDeserializer.deserializeEntityId(jsonObject, + StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_ID, + EntityTypeJsonDeserializer.deserializeEntityType(jsonObject, + StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_TYPE)); + } + + @Override + public Category getCategory() { + return Category.STREAM; + } + + @Override + public EntityId getEntityId() { + return entityId; + } + + @Override + public EntityType getEntityType() { + return entityId.getEntityType(); + } + + @Override + public JsonPointer getResourcePath() { + return resourcePath; + } + + @Override + public String getResourceType() { + return getEntityType().toString(); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, + final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + final Predicate predicate = schemaVersion.and(thePredicate); + jsonObjectBuilder.set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_TYPE, + entityId.getEntityType().toString(), predicate); + jsonObjectBuilder.set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_ID, + entityId.toString(), predicate); + jsonObjectBuilder.set(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH, + resourcePath.toString(), predicate); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), entityId, resourcePath); + } + + @SuppressWarnings({"squid:MethodCyclomaticComplexity", "squid:S1067", "OverlyComplexMethod"}) + @Override + public boolean equals(@Nullable final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final AbstractStreamingSubscriptionCommand other = (AbstractStreamingSubscriptionCommand) obj; + + return other.canEqual(this) && + super.equals(other) && + Objects.equals(entityId, other.entityId) && + Objects.equals(resourcePath, other.resourcePath); + } + + @Override + protected boolean canEqual(@Nullable final Object other) { + return other instanceof AbstractStreamingSubscriptionCommand; + } + + @Override + public String toString() { + return super.toString() + + ", entityId=" + entityId + + ", entityType=" + getEntityType() + + ", resourcePath=" + resourcePath; + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscription.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscription.java new file mode 100755 index 00000000000..8586c5cd5fe --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscription.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import java.util.Objects; +import java.util.function.Predicate; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableCommand; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.commands.CommandJsonDeserializer; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * Command for cancelling a subscription of streaming results. + * Corresponds to the reactive-streams signal {@code Subscription#cancel()}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableCommand(typePrefix = StreamingSubscriptionCommand.TYPE_PREFIX, name = CancelStreamingSubscription.NAME) +public final class CancelStreamingSubscription extends AbstractStreamingSubscriptionCommand + implements WithStreamingSubscriptionId { + + /** + * Name of the command. + */ + public static final String NAME = "cancel"; + + /** + * Type of this command. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final String subscriptionId; + + private CancelStreamingSubscription(final EntityId entityId, + final JsonPointer resourcePath, + final String subscriptionId, + final DittoHeaders dittoHeaders) { + super(TYPE, entityId, resourcePath, dittoHeaders); + this.subscriptionId = subscriptionId; + } + + /** + * Returns a new instance of the command. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream. + * @param subscriptionId ID of the subscription to cancel. + * @param dittoHeaders the headers of the command. + * @return a new command to cancel a subscription. + * @throws NullPointerException if {@code dittoHeaders} is {@code null}. + */ + public static CancelStreamingSubscription of(final EntityId entityId, + final JsonPointer resourcePath, + final String subscriptionId, + final DittoHeaders dittoHeaders) { + return new CancelStreamingSubscription(entityId, resourcePath, subscriptionId, dittoHeaders); + } + + /** + * Creates a new {@code CancelStreamingSubscription} from a JSON object. + * + * @param jsonObject the JSON object of which the command is to be created. + * @param dittoHeaders the headers of the command. + * @return the command. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static CancelStreamingSubscription fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new CommandJsonDeserializer(TYPE, jsonObject).deserialize(() -> + new CancelStreamingSubscription(deserializeEntityId(jsonObject), + JsonPointer.of( + jsonObject.getValueOrThrow(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH)), + jsonObject.getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID), + dittoHeaders + ) + ); + } + + @Override + public String getSubscriptionId() { + return subscriptionId; + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + super.appendPayload(jsonObjectBuilder, schemaVersion, thePredicate); + jsonObjectBuilder.set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, subscriptionId); + } + + @Override + public CancelStreamingSubscription setDittoHeaders(final DittoHeaders dittoHeaders) { + return new CancelStreamingSubscription(entityId, resourcePath, subscriptionId, dittoHeaders); + } + + @Override + public boolean equals(@Nullable final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CancelStreamingSubscription)) { + return false; + } + if (!super.equals(o)) { + return false; + } + final CancelStreamingSubscription that = (CancelStreamingSubscription) o; + return Objects.equals(subscriptionId, that.subscriptionId); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), subscriptionId); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[" + + super.toString() + + ", subscriptionId=" + subscriptionId + + ']'; + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscription.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscription.java new file mode 100755 index 00000000000..4a72cde4204 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscription.java @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import java.util.Objects; +import java.util.function.Predicate; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableCommand; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.commands.CommandJsonDeserializer; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * Command for requesting items from a subscription of streaming results. + * Corresponds to the reactive-streams signal {@code Subscription#request(long)}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableCommand(typePrefix = StreamingSubscriptionCommand.TYPE_PREFIX, name = RequestFromStreamingSubscription.NAME) +public final class RequestFromStreamingSubscription extends AbstractStreamingSubscriptionCommand + implements WithStreamingSubscriptionId { + + /** + * Name of the command. + */ + public static final String NAME = "request"; + + /** + * Type of this command. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final String subscriptionId; + private final long demand; + + private RequestFromStreamingSubscription(final EntityId entityId, + final JsonPointer resourcePath, + final String subscriptionId, + final long demand, + final DittoHeaders dittoHeaders) { + + super(TYPE, entityId, resourcePath, dittoHeaders); + this.subscriptionId = subscriptionId; + this.demand = demand; + } + + /** + * Returns a new instance of the command. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream. + * @param subscriptionId ID of the subscription to request from. + * @param demand how many pages to request. + * @param dittoHeaders the headers of the command. + * @return a new command to request from a subscription. + * @throws NullPointerException if {@code dittoHeaders} is {@code null}. + */ + public static RequestFromStreamingSubscription of(final EntityId entityId, + final JsonPointer resourcePath, + final String subscriptionId, + final long demand, + final DittoHeaders dittoHeaders) { + return new RequestFromStreamingSubscription(entityId, resourcePath, subscriptionId, demand, dittoHeaders); + } + + /** + * Creates a new {@code RequestSubscription} from a JSON object. + * + * @param jsonObject the JSON object of which the command is to be created. + * @param dittoHeaders the headers of the command. + * @return the command. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static RequestFromStreamingSubscription fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new CommandJsonDeserializer(TYPE, jsonObject).deserialize(() -> + new RequestFromStreamingSubscription(deserializeEntityId(jsonObject), + JsonPointer.of( + jsonObject.getValueOrThrow(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH)), + jsonObject.getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID), + jsonObject.getValueOrThrow(JsonFields.DEMAND), + dittoHeaders + ) + ); + } + + + @Override + public String getSubscriptionId() { + return subscriptionId; + } + + + /** + * Returns the demand which is to be included in the JSON of the retrieved entity. + * + * @return the demand. + */ + public long getDemand() { + return demand; + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + super.appendPayload(jsonObjectBuilder, schemaVersion, thePredicate); + jsonObjectBuilder.set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, subscriptionId); + jsonObjectBuilder.set(JsonFields.DEMAND, demand); + } + + @Override + public String getResourceType() { + return getEntityType().toString(); + } + + @Override + public RequestFromStreamingSubscription setDittoHeaders(final DittoHeaders dittoHeaders) { + return new RequestFromStreamingSubscription(entityId, resourcePath, subscriptionId, demand, dittoHeaders); + } + + @Override + public boolean equals(@Nullable final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RequestFromStreamingSubscription)) { + return false; + } + if (!super.equals(o)) { + return false; + } + final RequestFromStreamingSubscription that = (RequestFromStreamingSubscription) o; + return Objects.equals(subscriptionId, that.subscriptionId) && demand == that.demand; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), subscriptionId, demand); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[" + + super.toString() + + ", subscriptionId=" + subscriptionId + + ", demand=" + demand + + ']'; + } + + /** + * JSON fields of this command. + */ + public static final class JsonFields { + + /** + * JSON field for number of pages demanded by this command. + */ + public static final JsonFieldDefinition DEMAND = JsonFactory.newLongFieldDefinition("demand"); + + JsonFields() { + throw new AssertionError(); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/StreamingSubscriptionCommand.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/StreamingSubscriptionCommand.java new file mode 100755 index 00000000000..f57d240d934 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/StreamingSubscriptionCommand.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.eclipse.ditto.base.model.json.FieldType.REGULAR; +import static org.eclipse.ditto.base.model.json.JsonSchemaVersion.V_2; + +import org.eclipse.ditto.base.model.entity.type.WithEntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.SignalWithEntityId; +import org.eclipse.ditto.base.model.signals.WithResource; +import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; + +/** + * Aggregates all {@link Command}s which request a stream (e.g. a {@code SourceRef}) of + * {@link org.eclipse.ditto.base.model.signals.Signal}s to subscribe for. + * + * @param the type of the implementing class. + * @since 3.2.0 + */ +public interface StreamingSubscriptionCommand> extends Command, + WithEntityType, SignalWithEntityId, WithResource { + + /** + * Resource type of streaming subscription commands. + */ + String RESOURCE_TYPE = "streaming.subscription"; + + /** + * Type Prefix of Streaming commands. + */ + String TYPE_PREFIX = RESOURCE_TYPE + "." + TYPE_QUALIFIER + ":"; + + @Override + default String getTypePrefix() { + return TYPE_PREFIX; + } + + @Override + T setDittoHeaders(DittoHeaders dittoHeaders); + + /** + * This class contains definitions for all specific fields of this command's JSON representation. + */ + final class JsonFields { + + private JsonFields() { + throw new AssertionError(); + } + + public static final JsonFieldDefinition JSON_ENTITY_ID = + JsonFactory.newStringFieldDefinition("entityId", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_ENTITY_TYPE = + JsonFactory.newStringFieldDefinition("entityType", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_RESOURCE_PATH = + JsonFactory.newStringFieldDefinition("resourcePath", REGULAR, V_2); + + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEvents.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEvents.java new file mode 100755 index 00000000000..aeca3a7738f --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEvents.java @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.eclipse.ditto.base.model.json.FieldType.REGULAR; +import static org.eclipse.ditto.base.model.json.JsonSchemaVersion.V_2; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableCommand; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * Command which starts a stream of journal entries as persisted events for a given EntityId. + * Corresponds to the reactive-streams signal {@code Publisher#subscribe(Subscriber)}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableCommand(typePrefix = StreamingSubscriptionCommand.TYPE_PREFIX, name = SubscribeForPersistedEvents.NAME) +public final class SubscribeForPersistedEvents extends AbstractStreamingSubscriptionCommand + implements StreamingSubscriptionCommand { + + /** + * The name of this streaming subscription command. + */ + public static final String NAME = "subscribeForPersistedEvents"; + + /** + * Type of this command. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final long fromHistoricalRevision; + private final long toHistoricalRevision; + + @Nullable private final Instant fromHistoricalTimestamp; + @Nullable private final Instant toHistoricalTimestamp; + @Nullable private final String prefix; + + private SubscribeForPersistedEvents(final EntityId entityId, + final JsonPointer resourcePath, + final long fromHistoricalRevision, + final long toHistoricalRevision, + @Nullable final Instant fromHistoricalTimestamp, + @Nullable final Instant toHistoricalTimestamp, + @Nullable final String prefix, + final DittoHeaders dittoHeaders) { + + super(TYPE, entityId, resourcePath, dittoHeaders); + this.fromHistoricalRevision = fromHistoricalRevision; + this.toHistoricalRevision = toHistoricalRevision; + this.fromHistoricalTimestamp = fromHistoricalTimestamp; + this.toHistoricalTimestamp = toHistoricalTimestamp; + this.prefix = prefix; + } + + /** + * Creates a new {@code SudoStreamSnapshots} command based on "from" and "to" {@code long} revisions. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream events. + * @param fromHistoricalRevision the revision to start the streaming from. + * @param toHistoricalRevision the revision to stop the streaming at. + * @param dittoHeaders the command headers of the request. + * @return the command. + * @throws NullPointerException if any non-nullable argument is {@code null}. + */ + public static SubscribeForPersistedEvents of(final EntityId entityId, + final JsonPointer resourcePath, + final long fromHistoricalRevision, + final long toHistoricalRevision, + final DittoHeaders dittoHeaders) { + + return new SubscribeForPersistedEvents(entityId, + resourcePath, + fromHistoricalRevision, + toHistoricalRevision, + null, + null, + null, + dittoHeaders); + } + + /** + * Creates a new {@code SudoStreamSnapshots} command based on "from" and "to" {@code Instant} timestamps. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream events. + * @param fromHistoricalTimestamp the timestamp to start the streaming from. + * @param toHistoricalTimestamp the timestamp to stop the streaming at. + * @param dittoHeaders the command headers of the request. + * @return the command. + * @throws NullPointerException if any non-nullable argument is {@code null}. + */ + public static SubscribeForPersistedEvents of(final EntityId entityId, + final JsonPointer resourcePath, + @Nullable final Instant fromHistoricalTimestamp, + @Nullable final Instant toHistoricalTimestamp, + final DittoHeaders dittoHeaders) { + + return new SubscribeForPersistedEvents(entityId, + resourcePath, + 0L, + Long.MAX_VALUE, + fromHistoricalTimestamp, + toHistoricalTimestamp, + null, + dittoHeaders); + } + + /** + * Creates a new {@code SudoStreamSnapshots} command based on "from" and "to" {@code Instant} timestamps. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream events. + * @param fromHistoricalRevision the revision to start the streaming from. + * @param toHistoricalRevision the revision to stop the streaming at. + * @param fromHistoricalTimestamp the timestamp to start the streaming from. + * @param toHistoricalTimestamp the timestamp to stop the streaming at. + * @param dittoHeaders the command headers of the request. + * @return the command. + * @throws NullPointerException if any non-nullable argument is {@code null}. + */ + public static SubscribeForPersistedEvents of(final EntityId entityId, + final JsonPointer resourcePath, + @Nullable final Long fromHistoricalRevision, + @Nullable final Long toHistoricalRevision, + @Nullable final Instant fromHistoricalTimestamp, + @Nullable final Instant toHistoricalTimestamp, + final DittoHeaders dittoHeaders) { + + return new SubscribeForPersistedEvents(entityId, + resourcePath, + null != fromHistoricalRevision ? fromHistoricalRevision : 0L, + null != toHistoricalRevision ? toHistoricalRevision : Long.MAX_VALUE, + fromHistoricalTimestamp, + toHistoricalTimestamp, + null, + dittoHeaders); + } + + /** + * Deserializes a {@code SubscribeForPersistedEvents} from the specified {@link JsonObject} argument. + * + * @param jsonObject the JSON object to be deserialized. + * @return the deserialized {@code SubscribeForPersistedEvents}. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if {@code jsonObject} did not contain all required + * fields. + * @throws org.eclipse.ditto.json.JsonParseException if {@code jsonObject} was not in the expected format. + */ + public static SubscribeForPersistedEvents fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new SubscribeForPersistedEvents(deserializeEntityId(jsonObject), + JsonPointer.of(jsonObject.getValueOrThrow(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH)), + jsonObject.getValueOrThrow(JsonFields.JSON_FROM_HISTORICAL_REVISION), + jsonObject.getValueOrThrow(JsonFields.JSON_TO_HISTORICAL_REVISION), + jsonObject.getValue(JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP).map(Instant::parse).orElse(null), + jsonObject.getValue(JsonFields.JSON_TO_HISTORICAL_TIMESTAMP).map(Instant::parse).orElse(null), + jsonObject.getValue(JsonFields.PREFIX).orElse(null), + dittoHeaders + ); + } + + /** + * Create a copy of this command with prefix set. The prefix is used to identify a streaming subscription manager + * if multiple are deployed in the cluster. + * + * @param prefix the subscription ID prefix. + * @return the new command. + */ + public SubscribeForPersistedEvents setPrefix(@Nullable final String prefix) { + return new SubscribeForPersistedEvents(entityId, resourcePath, fromHistoricalRevision, toHistoricalRevision, + fromHistoricalTimestamp, toHistoricalTimestamp, prefix, getDittoHeaders()); + } + + /** + * Returns the revision to start the streaming from. + * + * @return the revision to start the streaming from. + */ + public long getFromHistoricalRevision() { + return fromHistoricalRevision; + } + + /** + * Returns the timestamp to stop the streaming at. + * + * @return the timestamp to stop the streaming at. + */ + public long getToHistoricalRevision() { + return toHistoricalRevision; + } + + /** + * Returns the optional timestamp to start the streaming from. + * + * @return the optional timestamp to start the streaming from. + */ + public Optional getFromHistoricalTimestamp() { + return Optional.ofNullable(fromHistoricalTimestamp); + } + + /** + * Returns the optional timestamp to stop the streaming at. + * + * @return the optional timestamp to stop the streaming at. + */ + public Optional getToHistoricalTimestamp() { + return Optional.ofNullable(toHistoricalTimestamp); + } + + /** + * Get the prefix of subscription IDs. The prefix is used to identify a streaming subscription manager if multiple + * are deployed in the cluster. + * + * @return the subscription ID prefix. + */ + public Optional getPrefix() { + return Optional.ofNullable(prefix); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, + final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + super.appendPayload(jsonObjectBuilder, schemaVersion, thePredicate); + + final Predicate predicate = schemaVersion.and(thePredicate); + jsonObjectBuilder.set(JsonFields.JSON_FROM_HISTORICAL_REVISION, fromHistoricalRevision, predicate); + jsonObjectBuilder.set(JsonFields.JSON_TO_HISTORICAL_REVISION, toHistoricalRevision, predicate); + jsonObjectBuilder.set(JsonFields.JSON_TO_HISTORICAL_REVISION, toHistoricalRevision, predicate); + if (null != fromHistoricalTimestamp) { + jsonObjectBuilder.set(JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP, fromHistoricalTimestamp.toString(), + predicate); + } + if (null != toHistoricalTimestamp) { + jsonObjectBuilder.set(JsonFields.JSON_TO_HISTORICAL_TIMESTAMP, toHistoricalTimestamp.toString(), predicate); + } + getPrefix().ifPresent(thePrefix -> jsonObjectBuilder.set(JsonFields.PREFIX, thePrefix)); + } + + @Override + public String getTypePrefix() { + return TYPE_PREFIX; + } + + @Override + public SubscribeForPersistedEvents setDittoHeaders(final DittoHeaders dittoHeaders) { + return new SubscribeForPersistedEvents(entityId, resourcePath, fromHistoricalRevision, toHistoricalRevision, + fromHistoricalTimestamp, toHistoricalTimestamp, prefix, dittoHeaders); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), entityId, resourcePath, fromHistoricalRevision, toHistoricalRevision, + fromHistoricalTimestamp, toHistoricalTimestamp, prefix); + } + + @Override + public boolean equals(@Nullable final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final SubscribeForPersistedEvents that = (SubscribeForPersistedEvents) obj; + + return that.canEqual(this) && super.equals(that) && + fromHistoricalRevision == that.fromHistoricalRevision && + toHistoricalRevision == that.toHistoricalRevision && + Objects.equals(fromHistoricalTimestamp, that.fromHistoricalTimestamp) && + Objects.equals(toHistoricalTimestamp, that.toHistoricalTimestamp) && + Objects.equals(prefix, that.prefix); + } + + @Override + protected boolean canEqual(@Nullable final Object other) { + return other instanceof SubscribeForPersistedEvents; + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + + ", fromHistoricalRevision=" + fromHistoricalRevision + + ", toHistoricalRevision=" + toHistoricalRevision + + ", fromHistoricalTimestamp=" + fromHistoricalTimestamp + + ", toHistoricalTimestamp=" + toHistoricalTimestamp + + ", prefix=" + prefix + + "]"; + } + + /** + * This class contains definitions for all specific fields of this command's JSON representation. + */ + public static final class JsonFields { + + private JsonFields() { + throw new AssertionError(); + } + + public static final JsonFieldDefinition JSON_FROM_HISTORICAL_REVISION = + JsonFactory.newLongFieldDefinition("fromHistoricalRevision", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_TO_HISTORICAL_REVISION = + JsonFactory.newLongFieldDefinition("toHistoricalRevision", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_FROM_HISTORICAL_TIMESTAMP = + JsonFactory.newStringFieldDefinition("fromHistoricalTimestamp", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_TO_HISTORICAL_TIMESTAMP = + JsonFactory.newStringFieldDefinition("toHistoricalTimestamp", REGULAR, V_2); + + static final JsonFieldDefinition PREFIX = + JsonFactory.newStringFieldDefinition("prefix", REGULAR, V_2); + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/package-info.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/package-info.java new file mode 100755 index 00000000000..565442a816b --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/package-info.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.eclipse.ditto.utils.jsr305.annotations.AllValuesAreNonnullByDefault +package org.eclipse.ditto.base.model.signals.commands.streaming; + diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEvent.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEvent.java index 3f19feec46a..815c280930a 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEvent.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEvent.java @@ -98,6 +98,7 @@ public String getManifest() { @Override public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate thePredicate) { final Predicate predicate = schemaVersion.and(thePredicate); + final JsonObjectBuilder jsonObjectBuilder = JsonFactory.newObjectBuilder() // TYPE is included unconditionally: .set(JsonFields.TYPE, type) diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEventsourcedEvent.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEventsourcedEvent.java index af20d1b37e7..0276fa4d5e0 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEventsourcedEvent.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEventsourcedEvent.java @@ -96,6 +96,7 @@ public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate< // it shall not invoke super.toJson(...) because in that case "appendPayloadAndBuild" would be invoked twice // and the order of the fields to appear in the JSON would not be controllable! final Predicate predicate = schemaVersion.and(thePredicate); + final JsonObjectBuilder jsonObjectBuilder = JsonFactory.newObjectBuilder() // TYPE + entityId is included unconditionally: .set(Event.JsonFields.TYPE, getType()) diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/EventJsonDeserializer.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/EventJsonDeserializer.java index 782ea697881..e8704a716b9 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/EventJsonDeserializer.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/EventJsonDeserializer.java @@ -22,13 +22,13 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; +import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.exceptions.DittoJsonException; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonMissingFieldException; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonParseException; import org.eclipse.ditto.json.JsonValue; -import org.eclipse.ditto.base.model.entity.metadata.Metadata; -import org.eclipse.ditto.base.model.exceptions.DittoJsonException; /** * This class helps to deserialize JSON to a sub-class of {@link Event}. Hereby this class extracts the values which diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/AbstractStreamingSubscriptionEvent.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/AbstractStreamingSubscriptionEvent.java new file mode 100755 index 00000000000..ce0089bc51d --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/AbstractStreamingSubscriptionEvent.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.EntityIdJsonDeserializer; +import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.entity.type.EntityTypeJsonDeserializer; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; + +/** + * Abstract base class of subscription events. Package-private. Not to be extended in user code. + * + * @param the type of the implementing class. + * @since 3.2.0 + */ +@Immutable +abstract class AbstractStreamingSubscriptionEvent> implements + StreamingSubscriptionEvent { + + private final String type; + private final String subscriptionId; + + private final EntityId entityId; + private final DittoHeaders dittoHeaders; + + /** + * Constructs a new {@code AbstractStreamingSubscriptionEvent} object. + * + * @param type the type of this event. + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @throws NullPointerException if any argument but {@code timestamp} is {@code null}. + */ + protected AbstractStreamingSubscriptionEvent(final String type, + final String subscriptionId, + final EntityId entityId, + final DittoHeaders dittoHeaders) { + + this.type = checkNotNull(type, "type"); + this.subscriptionId = checkNotNull(subscriptionId, "subscriptionId"); + this.entityId = checkNotNull(entityId, "entityId"); + this.dittoHeaders = checkNotNull(dittoHeaders, "dittoHeaders"); + } + + @Override + public String getSubscriptionId() { + return subscriptionId; + } + + @Override + public String getType() { + return type; + } + + @Override + public EntityId getEntityId() { + return entityId; + } + + @Override + public EntityType getEntityType() { + return entityId.getEntityType(); + } + + @Override + public Optional getTimestamp() { + // subscription events have no timestamp. + return Optional.empty(); + } + + @Override + public Optional getMetadata() { + return Optional.empty(); + } + + @Override + public DittoHeaders getDittoHeaders() { + return dittoHeaders; + } + + @Nonnull + @Override + public String getManifest() { + return getType(); + } + + @Override + public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate thePredicate) { + final JsonObjectBuilder jsonObjectBuilder = JsonFactory.newObjectBuilder() + // TYPE is included unconditionally + .set(Event.JsonFields.TYPE, type) + .set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, subscriptionId) + .set(StreamingSubscriptionEvent.JsonFields.JSON_ENTITY_ID, entityId.toString()) + .set(StreamingSubscriptionEvent.JsonFields.JSON_ENTITY_TYPE, entityId.getEntityType().toString()); + + appendPayload(jsonObjectBuilder); + + return jsonObjectBuilder.build(); + } + + protected static EntityId deserializeEntityId(final JsonObject jsonObject) { + return EntityIdJsonDeserializer.deserializeEntityId(jsonObject, + StreamingSubscriptionEvent.JsonFields.JSON_ENTITY_ID, + EntityTypeJsonDeserializer.deserializeEntityType(jsonObject, + StreamingSubscriptionEvent.JsonFields.JSON_ENTITY_TYPE)); + } + + /** + * Appends the event specific custom payload to the passed {@code jsonObjectBuilder}. + * + * @param jsonObjectBuilder the JsonObjectBuilder to add the custom payload to. + */ + protected abstract void appendPayload(final JsonObjectBuilder jsonObjectBuilder); + + @Override + public boolean equals(@Nullable final Object o) { + if (o != null && getClass() == o.getClass()) { + final AbstractStreamingSubscriptionEvent that = (AbstractStreamingSubscriptionEvent) o; + return Objects.equals(type, that.type) && + Objects.equals(subscriptionId, that.subscriptionId) && + Objects.equals(entityId, that.entityId) && + Objects.equals(dittoHeaders, that.dittoHeaders); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(type, subscriptionId, entityId, dittoHeaders); + } + + @Override + public String toString() { + return "type=" + type + + ", subscriptionId=" + subscriptionId + + ", entityId=" + entityId + + ", dittoHeaders=" + dittoHeaders; + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionComplete.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionComplete.java new file mode 100755 index 00000000000..e6bd039cddd --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionComplete.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableEvent; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * This event is emitted after all items of a subscription are sent. + * Corresponds to the reactive-streams signal {@code Subscriber#onComplete()}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableEvent(name = StreamingSubscriptionComplete.NAME, typePrefix = StreamingSubscriptionEvent.TYPE_PREFIX) +public final class StreamingSubscriptionComplete + extends AbstractStreamingSubscriptionEvent { + + /** + * Name of the event. + */ + public static final String NAME = "complete"; + + /** + * Type of this event. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private StreamingSubscriptionComplete(final String subscriptionId, final EntityId entityId, + final DittoHeaders dittoHeaders) { + super(TYPE, subscriptionId, entityId, dittoHeaders); + } + + /** + * Constructs a new {@code StreamingSubscriptionComplete} object. + * + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the StreamingSubscriptionComplete created. + * @throws NullPointerException if either argument is null. + */ + public static StreamingSubscriptionComplete of(final String subscriptionId, final EntityId entityId, + final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionComplete(subscriptionId, entityId, dittoHeaders); + } + + /** + * Creates a new {@code StreamingSubscriptionComplete} from a JSON object. + * + * @param jsonObject the JSON object from which a new StreamingSubscriptionComplete instance is to be created. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the {@code StreamingSubscriptionComplete} which was created from the given JSON object. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionComplete fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new EventJsonDeserializer(TYPE, jsonObject) + .deserialize((revision, timestamp, metadata) -> { + final String subscriptionId = jsonObject + .getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID); + final EntityId entityId = deserializeEntityId(jsonObject); + return new StreamingSubscriptionComplete(subscriptionId, entityId, dittoHeaders); + }); + } + + @Override + public JsonPointer getResourcePath() { + return JsonPointer.empty(); + } + + @Override + public StreamingSubscriptionComplete setDittoHeaders(final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionComplete(getSubscriptionId(), getEntityId(), dittoHeaders); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder) { + // nothing to add + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + "]"; + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreated.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreated.java new file mode 100755 index 00000000000..6c6ffc518e2 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreated.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableEvent; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * This event is emitted after a stream is established for items to be streamed in the back-end. + * Corresponds to the reactive-streams signal {@code Subscriber#onSubscribe(Subscription)}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableEvent(name = StreamingSubscriptionCreated.NAME, typePrefix = StreamingSubscriptionEvent.TYPE_PREFIX) +public final class StreamingSubscriptionCreated + extends AbstractStreamingSubscriptionEvent { + + /** + * Name of the event. + */ + public static final String NAME = "created"; + + /** + * Type of this event. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private StreamingSubscriptionCreated(final String subscriptionId, + final EntityId entityId, + final DittoHeaders dittoHeaders) { + super(TYPE, subscriptionId, entityId, dittoHeaders); + } + + /** + * Constructs a new {@code StreamingSubscriptionCreated} event. + * + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the event. + * @throws NullPointerException if either argument is null. + */ + public static StreamingSubscriptionCreated of(final String subscriptionId, + final EntityId entityId, + final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionCreated(subscriptionId, entityId, dittoHeaders); + } + + /** + * Creates a new {@code StreamingSubscriptionCreated} from a JSON object. + * + * @param jsonObject the JSON object from which a new StreamingSubscriptionCreated instance is to be created. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the {@code StreamingSubscriptionCreated} which was created from the given JSON object. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionCreated fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new EventJsonDeserializer(TYPE, jsonObject) + .deserialize((revision, timestamp, metadata) -> { + final String subscriptionId = jsonObject + .getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID); + final EntityId entityId = deserializeEntityId(jsonObject); + return new StreamingSubscriptionCreated(subscriptionId, entityId, dittoHeaders); + }); + } + + @Override + public JsonPointer getResourcePath() { + return JsonPointer.empty(); + } + + @Override + public StreamingSubscriptionCreated setDittoHeaders(final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionCreated(getSubscriptionId(), getEntityId(), dittoHeaders); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder) { + // nothing to add + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + "]"; + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionEvent.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionEvent.java new file mode 100755 index 00000000000..a009d0fded2 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionEvent.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.eclipse.ditto.base.model.json.FieldType.REGULAR; +import static org.eclipse.ditto.base.model.json.JsonSchemaVersion.V_2; + +import org.eclipse.ditto.base.model.entity.id.WithEntityId; +import org.eclipse.ditto.base.model.entity.type.WithEntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; + +/** + * Interface for all outgoing messages related to a subscription for streaming something. + * + * @param the type of the implementing class. + * @since 3.2.0 + */ +public interface StreamingSubscriptionEvent> extends Event, + WithEntityId, WithEntityType { + + /** + * Resource type of streaming subscription events. + */ + String RESOURCE_TYPE = "streaming.subscription"; + + /** + * Type Prefix of Streaming events. + */ + String TYPE_PREFIX = RESOURCE_TYPE + "." + TYPE_QUALIFIER + ":"; + + /** + * Returns the subscriptionId identifying the session of this streaming signal. + * + * @return the subscriptionId. + */ + String getSubscriptionId(); + + @Override + T setDittoHeaders(DittoHeaders dittoHeaders); + + @Override + default String getResourceType() { + return RESOURCE_TYPE; + } + + /** + * This class contains definitions for all specific fields of this event's JSON representation. + */ + final class JsonFields { + + private JsonFields() { + throw new AssertionError(); + } + + public static final JsonFieldDefinition JSON_ENTITY_ID = + JsonFactory.newStringFieldDefinition("entityId", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_ENTITY_TYPE = + JsonFactory.newStringFieldDefinition("entityType", REGULAR, V_2); + + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailed.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailed.java new file mode 100755 index 00000000000..4e4f878e34d --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailed.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableEvent; +import org.eclipse.ditto.base.model.signals.GlobalErrorRegistry; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * This event is emitted after a stream failed. + * Corresponds to the reactive-streams signal {@code Subscriber#onError()}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableEvent(name = StreamingSubscriptionFailed.NAME, typePrefix = StreamingSubscriptionEvent.TYPE_PREFIX) +public final class StreamingSubscriptionFailed extends AbstractStreamingSubscriptionEvent { + + /** + * Name of the event. + */ + public static final String NAME = "failed"; + + /** + * Type of this event. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final DittoRuntimeException error; + + private StreamingSubscriptionFailed(final String subscriptionId, + final EntityId entityId, + final DittoRuntimeException error, + final DittoHeaders dittoHeaders) { + super(TYPE, subscriptionId, entityId, dittoHeaders); + this.error = error; + } + + /** + * Constructs a new {@code StreamingSubscriptionFailed} object. + * + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param error the cause of the failure. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the StreamingSubscriptionFailed created. + * @throws NullPointerException if either argument is null. + */ + public static StreamingSubscriptionFailed of(final String subscriptionId, + final EntityId entityId, + final DittoRuntimeException error, + final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionFailed(subscriptionId, entityId, error, dittoHeaders); + } + + /** + * Creates a new {@code StreamingSubscriptionFailed} from a JSON object. + * + * @param jsonObject the JSON object from which a new StreamingSubscriptionFailed instance is to be created. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the {@code StreamingSubscriptionFailed} which was created from the given JSON object. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionFailed fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new EventJsonDeserializer(TYPE, jsonObject) + .deserialize((revision, timestamp, metadata) -> { + final String subscriptionId = + jsonObject.getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID); + final JsonObject errorJson = jsonObject.getValueOrThrow(JsonFields.ERROR); + final EntityId entityId = deserializeEntityId(jsonObject); + final DittoRuntimeException error = + GlobalErrorRegistry.getInstance().parse(errorJson, dittoHeaders); + return new StreamingSubscriptionFailed(subscriptionId, entityId, error, dittoHeaders); + }); + } + + /** + * Get the cause of the failure. + * + * @return the error in JSON format. + */ + public DittoRuntimeException getError() { + return error; + } + + /** + * Create a copy of this event with a new error. + * + * @param error the new error. + * @return the copied event with new error. + */ + public StreamingSubscriptionFailed setError(final DittoRuntimeException error) { + return new StreamingSubscriptionFailed(getSubscriptionId(), getEntityId(), error, getDittoHeaders()); + } + + @Override + public JsonPointer getResourcePath() { + return JsonPointer.empty(); + } + + @Override + public StreamingSubscriptionFailed setDittoHeaders(final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionFailed(getSubscriptionId(), getEntityId(), error, dittoHeaders); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder) { + jsonObjectBuilder.set(JsonFields.ERROR, error.toJson()); + } + + @Override + public boolean equals(final Object o) { + return super.equals(o) && Objects.equals(error, ((StreamingSubscriptionFailed) o).error); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), error); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + ", error=" + error + "]"; + } + + /** + * Json fields of this event. + */ + public static final class JsonFields { + + /** + * Json fields for a JSON representation of the error. + */ + public static final JsonFieldDefinition ERROR = + JsonFactory.newJsonObjectFieldDefinition("error"); + + JsonFields() { + throw new AssertionError(); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNext.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNext.java new file mode 100755 index 00000000000..c8539110fb9 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNext.java @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableEvent; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.json.JsonValue; + +/** + * This event is emitted after the next items to stream are ready. + * Corresponds to the reactive-streams signal {@code Subscriber#onNext(T)}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableEvent(name = StreamingSubscriptionHasNext.NAME, typePrefix = StreamingSubscriptionEvent.TYPE_PREFIX) +public final class StreamingSubscriptionHasNext + extends AbstractStreamingSubscriptionEvent { + + /** + * Name of the event. + */ + public static final String NAME = "next"; + + /** + * Type of this event. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final JsonValue item; + + private StreamingSubscriptionHasNext(final String subscriptionId, + final EntityId entityId, + final JsonValue item, + final DittoHeaders dittoHeaders) { + super(TYPE, subscriptionId, entityId, dittoHeaders); + this.item = item; + } + + /** + * Constructs a new {@code SubscriptionHasNext} object. + * + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param item the "next" item. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the SubscriptionHasNext created. + * @throws NullPointerException if either argument is null. + */ + public static StreamingSubscriptionHasNext of(final String subscriptionId, + final EntityId entityId, + final JsonValue item, + final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionHasNext(subscriptionId, entityId, item, dittoHeaders); + } + + /** + * Creates a new {@code SubscriptionHasNext} from a JSON object. + * + * @param jsonObject the JSON object from which a new SubscriptionHasNext instance is to be created. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the {@code SubscriptionHasNext} which was created from the given JSON object. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionHasNext fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new EventJsonDeserializer(TYPE, jsonObject) + .deserialize((revision, timestamp, metadata) -> { + final String subscriptionId = + jsonObject.getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID); + final EntityId entityId = deserializeEntityId(jsonObject); + final JsonValue item = jsonObject.getValueOrThrow(JsonFields.ITEM); + return new StreamingSubscriptionHasNext(subscriptionId, entityId, item, dittoHeaders); + }); + } + + /** + * Get the "next" item. + * + * @return the next item. + */ + public JsonValue getItem() { + return item; + } + + /** + * Create a copy of this event with a new item. + * + * @param item the new item. + * @return the copied event with new item. + */ + public StreamingSubscriptionHasNext setItem(final JsonValue item) { + return new StreamingSubscriptionHasNext(getSubscriptionId(), getEntityId(), item, getDittoHeaders()); + } + + @Override + public JsonPointer getResourcePath() { + return JsonPointer.empty(); + } + + @Override + public StreamingSubscriptionHasNext setDittoHeaders(final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionHasNext(getSubscriptionId(), getEntityId(), item, dittoHeaders); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder) { + jsonObjectBuilder.set(JsonFields.ITEM, item); + } + + @Override + public boolean equals(final Object o) { + // super.equals(o) guarantees getClass() == o.getClass() + return super.equals(o) && Objects.equals(item, ((StreamingSubscriptionHasNext) o).item); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), item); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + ", item=" + item + "]"; + } + + /** + * Json fields of this event. + */ + public static final class JsonFields { + + /** + * Json field for "next" item. + */ + public static final JsonFieldDefinition ITEM = + JsonFactory.newJsonValueFieldDefinition("item"); + + JsonFields() { + throw new AssertionError(); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/package-info.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/package-info.java new file mode 100755 index 00000000000..7a13cf46a93 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/package-info.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.eclipse.ditto.utils.jsr305.annotations.AllValuesAreNonnullByDefault +package org.eclipse.ditto.base.model.signals.events.streaming; + diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java index 41d134522dc..be15e71c7f9 100644 --- a/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java @@ -21,6 +21,7 @@ public final class LikeHelperTest { @Test public void testWildcards() { + //case sensitive test cases assertExpression("", "*", true); assertExpression("foo", "*", true); assertExpression("foo.bar", "foo.bar", true); @@ -35,4 +36,23 @@ private static void assertExpression(final String value, final String expression Pattern p = Pattern.compile(LikeHelper.convertToRegexSyntax(expression)); Assert.assertEquals(matches, p.matcher(value).matches()); } + + @Test + public void testCaseInsensitiveWildcards() { + //case insensitive test cases + assertExpressionCaseInsensitive("", "*", true); + assertExpressionCaseInsensitive("foo", "*", true); + assertExpressionCaseInsensitive("foo.bar", "FOO.BAR", true); + assertExpressionCaseInsensitive("foo..bar", "foo.bar", false); + assertExpressionCaseInsensitive("foo..bar", "FOO*", true); + assertExpressionCaseInsensitive("foo..bar", "*Bar", true); + assertExpressionCaseInsensitive("foo.bar.baz", "bar", false); + assertExpressionCaseInsensitive("foo.bar.baz", "*bAr*", true); + } + + private static void assertExpressionCaseInsensitive(final String value, final String expression, final boolean matches) { + Pattern p = Pattern.compile(LikeHelper.convertToRegexSyntax(expression), Pattern.CASE_INSENSITIVE); + Assert.assertEquals(matches, p.matcher(value).matches()); + } + } diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java index 5c53e597a8c..474b7ca73da 100755 --- a/base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java @@ -19,11 +19,13 @@ import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; import java.time.Duration; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -77,6 +79,7 @@ public final class ImmutableDittoHeadersTest { EntityTagMatchers.fromCommaSeparatedString("\"oneValue\",\"anotherValue\""); private static final EntityTagMatchers KNOWN_IF_NONE_MATCH = EntityTagMatchers.fromCommaSeparatedString("\"notOneValue\",\"notAnotherValue\""); + private static final IfEqual KNOWN_IF_EQUAL_OPTION = IfEqual.SKIP; private static final EntityTag KNOWN_ETAG = EntityTag.fromString("\"-12124212\""); private static final Collection KNOWN_READ_GRANTED_SUBJECTS = Lists.list(AuthorizationModelFactory.newAuthSubject("knownGrantedSubject1"), @@ -129,6 +132,12 @@ public final class ImmutableDittoHeadersTest { .build()) .build()) .build(); + private static final Long KNOWN_AT_HISTORICAL_REVISION = 42L; + private static final Instant KNOWN_AT_HISTORICAL_TIMESTAMP = Instant.now(); + + private static final JsonObject KNOWN_HISTORICAL_HEADERS = JsonObject.newBuilder() + .set(DittoHeaderDefinition.ORIGINATOR.getKey(), "foo:bar") + .build(); static { @@ -164,6 +173,7 @@ public void settingAllKnownHeadersWorksAsExpected() { .eTag(KNOWN_ETAG) .ifMatch(KNOWN_IF_MATCH) .ifNoneMatch(KNOWN_IF_NONE_MATCH) + .ifEqual(KNOWN_IF_EQUAL_OPTION) .origin(KNOWN_ORIGIN) .contentType(KNOWN_CONTENT_TYPE) .replyTarget(Integer.valueOf(KNOWN_REPLY_TARGET)) @@ -199,6 +209,9 @@ public void settingAllKnownHeadersWorksAsExpected() { .putHeader(DittoHeaderDefinition.GET_METADATA.getKey(), KNOWN_DITTO_GET_METADATA ) .putHeader(DittoHeaderDefinition.DELETE_METADATA.getKey(), KNOWN_DITTO_DELETE_METADATA ) .putHeader(DittoHeaderDefinition.DITTO_METADATA.getKey(), KNOWN_DITTO_METADATA.formatAsString()) + .putHeader(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION)) + .putHeader(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP)) + .putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString()) .build(); assertThat(underTest).isEqualTo(expectedHeaderMap); @@ -491,6 +504,7 @@ public void toJsonReturnsExpected() { authorizationSubjectsToJsonArray(KNOWN_READ_REVOKED_SUBJECTS)) .set(DittoHeaderDefinition.IF_MATCH.getKey(), KNOWN_IF_MATCH.toString()) .set(DittoHeaderDefinition.IF_NONE_MATCH.getKey(), KNOWN_IF_NONE_MATCH.toString()) + .set(DittoHeaderDefinition.IF_EQUAL.getKey(), KNOWN_IF_EQUAL_OPTION.toString()) .set(DittoHeaderDefinition.ETAG.getKey(), KNOWN_ETAG.toString()) .set(DittoHeaderDefinition.ORIGIN.getKey(), KNOWN_ORIGIN) .set(DittoHeaderDefinition.CONTENT_TYPE.getKey(), KNOWN_CONTENT_TYPE) @@ -526,6 +540,9 @@ public void toJsonReturnsExpected() { .set(DittoHeaderDefinition.GET_METADATA.getKey(), KNOWN_DITTO_GET_METADATA) .set(DittoHeaderDefinition.DELETE_METADATA.getKey(), KNOWN_DITTO_DELETE_METADATA) .set(DittoHeaderDefinition.DITTO_METADATA.getKey(), KNOWN_DITTO_METADATA) + .set(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), KNOWN_AT_HISTORICAL_REVISION) + .set(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), KNOWN_AT_HISTORICAL_TIMESTAMP.toString()) + .set(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS) .build(); final Map allKnownHeaders = createMapContainingAllKnownHeaders(); @@ -710,7 +727,7 @@ public void preserveCapitalizationOfCorrelationId() { } private static Map createMapContainingAllKnownHeaders() { - final Map result = new HashMap<>(); + final Map result = new LinkedHashMap<>(); result.put(DittoHeaderDefinition.AUTHORIZATION_CONTEXT.getKey(), AUTH_CONTEXT.toJsonString()); result.put(DittoHeaderDefinition.CORRELATION_ID.getKey(), KNOWN_CORRELATION_ID); result.put(DittoHeaderDefinition.SCHEMA_VERSION.getKey(), KNOWN_SCHEMA_VERSION.toString()); @@ -725,6 +742,7 @@ private static Map createMapContainingAllKnownHeaders() { authorizationSubjectsToJsonArray(KNOWN_READ_REVOKED_SUBJECTS).toString()); result.put(DittoHeaderDefinition.IF_MATCH.getKey(), KNOWN_IF_MATCH.toString()); result.put(DittoHeaderDefinition.IF_NONE_MATCH.getKey(), KNOWN_IF_NONE_MATCH.toString()); + result.put(DittoHeaderDefinition.IF_EQUAL.getKey(), KNOWN_IF_EQUAL_OPTION.toString()); result.put(DittoHeaderDefinition.ETAG.getKey(), KNOWN_ETAG.toString()); result.put(DittoHeaderDefinition.CONTENT_TYPE.getKey(), KNOWN_CONTENT_TYPE); result.put(DittoHeaderDefinition.ACCEPT.getKey(), KNOWN_ACCEPT); @@ -762,6 +780,9 @@ private static Map createMapContainingAllKnownHeaders() { result.put(DittoHeaderDefinition.GET_METADATA.getKey(), KNOWN_DITTO_GET_METADATA); result.put(DittoHeaderDefinition.DELETE_METADATA.getKey(), KNOWN_DITTO_DELETE_METADATA); result.put(DittoHeaderDefinition.DITTO_METADATA.getKey(), KNOWN_DITTO_METADATA.formatAsString()); + result.put(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION)); + result.put(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP)); + result.put(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString()); return result; } diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/ShardedMessageEnvelopeTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/ShardedMessageEnvelopeTest.java index ccc35d53b85..b1e2ad591f2 100644 --- a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/ShardedMessageEnvelopeTest.java +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/ShardedMessageEnvelopeTest.java @@ -14,11 +14,12 @@ import static org.assertj.core.api.Assertions.assertThat; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.entity.type.EntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObject; import org.junit.Test; import nl.jqno.equalsverifier.EqualsVerifier; @@ -31,7 +32,7 @@ public final class ShardedMessageEnvelopeTest { private static final DittoHeaders DITTO_HEADERS = DittoHeaders.empty(); private static final EntityId MESSAGE_ID = - EntityId.of(EntityType.of("thing"), "org.eclipse.ditto.test:thingId"); + NamespacedEntityId.of(EntityType.of("thing"), "org.eclipse.ditto.test:thingId"); private static final String TYPE = "message-type"; private static final JsonObject MESSAGE = JsonFactory.newObjectBuilder().set("hello", "world").build(); diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscriptionTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscriptionTest.java new file mode 100755 index 00000000000..520e43134e2 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscriptionTest.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonPointer; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link CancelStreamingSubscription}. + */ +public final class CancelStreamingSubscriptionTest { + + @Test + public void assertImmutability() { + assertInstancesOf(CancelStreamingSubscription.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(CancelStreamingSubscription.class) + .withRedefinedSuperclass() + .verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final CancelStreamingSubscription underTest = CancelStreamingSubscription.of( + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), + JsonPointer.of("/"), + UUID.randomUUID().toString(), dittoHeaders); + final CancelStreamingSubscription deserialized = CancelStreamingSubscription.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } + +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscriptionTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscriptionTest.java new file mode 100755 index 00000000000..1c0c8f31988 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscriptionTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonPointer; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link RequestFromStreamingSubscription}. + */ +public final class RequestFromStreamingSubscriptionTest { + + @Test + public void assertImmutability() { + assertInstancesOf(RequestFromStreamingSubscription.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(RequestFromStreamingSubscription.class) + .withRedefinedSuperclass() + .verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final RequestFromStreamingSubscription + underTest = RequestFromStreamingSubscription.of(NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), + JsonPointer.of("/"), UUID.randomUUID().toString(), 9L, dittoHeaders); + final RequestFromStreamingSubscription deserialized = RequestFromStreamingSubscription.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } + +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEventsTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEventsTest.java new file mode 100755 index 00000000000..c78c8dab609 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEventsTest.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.AllowedReason.provided; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.time.Instant; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonPointer; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link SubscribeForPersistedEvents}. + */ +public final class SubscribeForPersistedEventsTest { + + private static final String KNOWN_ENTITY_ID_STR = "foo:bar"; + private static final String KNOWN_ENTITY_TYPE_STR = "thing"; + private static final String KNOWN_RESOURCE_PATH = "/"; + + private static final long KNOWN_FROM_REV = 23L; + private static final long KNOWN_TO_REV = 42L; + private static final String KNOWN_FROM_TS = "2022-10-25T14:00:00Z"; + private static final String KNOWN_TO_TS = "2022-10-25T15:00:00Z"; + + private static final String JSON_ALL_FIELDS = JsonFactory.newObjectBuilder() + .set(Command.JsonFields.TYPE, SubscribeForPersistedEvents.TYPE) + .set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_TYPE, KNOWN_ENTITY_TYPE_STR) + .set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_ID, KNOWN_ENTITY_ID_STR) + .set(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH, KNOWN_RESOURCE_PATH) + .set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_REVISION, KNOWN_FROM_REV) + .set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_REVISION, KNOWN_TO_REV) + .set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP, KNOWN_FROM_TS) + .set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_TIMESTAMP, KNOWN_TO_TS) + .build() + .toString(); + + private static final String JSON_MINIMAL = JsonFactory.newObjectBuilder() + .set(Command.JsonFields.TYPE, SubscribeForPersistedEvents.TYPE) + .set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_TYPE, KNOWN_ENTITY_TYPE_STR) + .set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_ID, KNOWN_ENTITY_ID_STR) + .set(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH, KNOWN_RESOURCE_PATH) + .set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_REVISION, KNOWN_FROM_REV) + .set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_REVISION, KNOWN_TO_REV) + .build().toString(); + + @Test + public void assertImmutability() { + assertInstancesOf(SubscribeForPersistedEvents.class, + areImmutable(), + provided(Instant.class).isAlsoImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(SubscribeForPersistedEvents.class) + .withRedefinedSuperclass() + .verify(); + } + + @Test + public void toJsonWithAllFieldsSet() { + final SubscribeForPersistedEvents command = SubscribeForPersistedEvents.of( + NamespacedEntityId.of(EntityType.of(KNOWN_ENTITY_TYPE_STR), KNOWN_ENTITY_ID_STR), + JsonPointer.of(KNOWN_RESOURCE_PATH), + KNOWN_FROM_REV, + KNOWN_TO_REV, + Instant.parse(KNOWN_FROM_TS), + Instant.parse(KNOWN_TO_TS), + DittoHeaders.empty() + ); + + final String json = command.toJsonString(); + assertThat(json).isEqualTo(JSON_ALL_FIELDS); + } + + @Test + public void toJsonWithOnlyRequiredFieldsSet() { + final SubscribeForPersistedEvents command = SubscribeForPersistedEvents.of( + NamespacedEntityId.of(EntityType.of(KNOWN_ENTITY_TYPE_STR), KNOWN_ENTITY_ID_STR), + JsonPointer.of(KNOWN_RESOURCE_PATH), + KNOWN_FROM_REV, + KNOWN_TO_REV, + DittoHeaders.empty()); + final String json = command.toJsonString(); + assertThat(json).isEqualTo(JSON_MINIMAL); + } + + @Test + public void fromJsonWithAllFieldsSet() { + final SubscribeForPersistedEvents command = SubscribeForPersistedEvents.of( + NamespacedEntityId.of(EntityType.of(KNOWN_ENTITY_TYPE_STR), KNOWN_ENTITY_ID_STR), + JsonPointer.of(KNOWN_RESOURCE_PATH), + KNOWN_FROM_REV, + KNOWN_TO_REV, + Instant.parse(KNOWN_FROM_TS), + Instant.parse(KNOWN_TO_TS), + DittoHeaders.empty() + ); + assertThat(SubscribeForPersistedEvents.fromJson(JsonObject.of(JSON_ALL_FIELDS), DittoHeaders.empty())) + .isEqualTo(command); + } + + @Test + public void fromJsonWithOnlyRequiredFieldsSet() { + assertThat(SubscribeForPersistedEvents.fromJson(JsonObject.of(JSON_MINIMAL), DittoHeaders.empty())) + .isEqualTo(SubscribeForPersistedEvents.of( + NamespacedEntityId.of(EntityType.of(KNOWN_ENTITY_TYPE_STR), KNOWN_ENTITY_ID_STR), + JsonPointer.of(KNOWN_RESOURCE_PATH), + KNOWN_FROM_REV, + KNOWN_TO_REV, + DittoHeaders.empty())); + } + +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCompleteTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCompleteTest.java new file mode 100644 index 00000000000..1f628cf250f --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCompleteTest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link StreamingSubscriptionComplete}. + */ +public final class StreamingSubscriptionCompleteTest { + + @Test + public void assertImmutability() { + assertInstancesOf(StreamingSubscriptionComplete.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(StreamingSubscriptionComplete.class).withRedefinedSuperclass().verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final StreamingSubscriptionComplete underTest = StreamingSubscriptionComplete.of(UUID.randomUUID().toString(), + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), dittoHeaders); + final StreamingSubscriptionComplete deserialized = StreamingSubscriptionComplete.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreatedTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreatedTest.java new file mode 100644 index 00000000000..cc97fc9cee3 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreatedTest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link StreamingSubscriptionCreated}. + */ +public final class StreamingSubscriptionCreatedTest { + + @Test + public void assertImmutability() { + assertInstancesOf(StreamingSubscriptionCreated.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(StreamingSubscriptionCreated.class).withRedefinedSuperclass().verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final StreamingSubscriptionCreated underTest = StreamingSubscriptionCreated.of(UUID.randomUUID().toString(), + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), dittoHeaders); + final StreamingSubscriptionCreated deserialized = StreamingSubscriptionCreated.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailedTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailedTest.java new file mode 100644 index 00000000000..03ccc64ab3b --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailedTest.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.AllowedReason.provided; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.InvalidRqlExpressionException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.GlobalErrorRegistry; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link StreamingSubscriptionFailed}. + */ +public final class StreamingSubscriptionFailedTest { + + @Test + public void assertImmutability() { + assertInstancesOf(StreamingSubscriptionFailed.class, areImmutable(), + provided(DittoRuntimeException.class).isAlsoImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(StreamingSubscriptionFailed.class).withRedefinedSuperclass().verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final DittoRuntimeException error = GlobalErrorRegistry.getInstance() + .parse(InvalidRqlExpressionException.newBuilder().build().toJson(), dittoHeaders); + final StreamingSubscriptionFailed underTest = StreamingSubscriptionFailed.of(UUID.randomUUID().toString(), + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), error, dittoHeaders); + final StreamingSubscriptionFailed deserialized = StreamingSubscriptionFailed.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNextTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNextTest.java new file mode 100644 index 00000000000..15c371b4567 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNextTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.AllowedReason.provided; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonArray; +import org.eclipse.ditto.json.JsonValue; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link StreamingSubscriptionHasNext}. + */ +public final class StreamingSubscriptionHasNextTest { + + @Test + public void assertImmutability() { + assertInstancesOf(StreamingSubscriptionHasNext.class, areImmutable(), provided(JsonValue.class).isAlsoImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(StreamingSubscriptionHasNext.class).withRedefinedSuperclass().verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final JsonArray items = JsonArray.of("[{\"x\":1},{\"x\":2}]"); + final StreamingSubscriptionHasNext + underTest = StreamingSubscriptionHasNext.of(UUID.randomUUID().toString(), + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), items, dittoHeaders); + final StreamingSubscriptionHasNext deserialized = StreamingSubscriptionHasNext.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } +} diff --git a/base/service/src/main/java/org/eclipse/ditto/base/service/config/limits/DefaultLimitsConfig.java b/base/service/src/main/java/org/eclipse/ditto/base/service/config/limits/DefaultLimitsConfig.java index ff45d6567ad..567320531fe 100644 --- a/base/service/src/main/java/org/eclipse/ditto/base/service/config/limits/DefaultLimitsConfig.java +++ b/base/service/src/main/java/org/eclipse/ditto/base/service/config/limits/DefaultLimitsConfig.java @@ -69,6 +69,11 @@ public long getPoliciesMaxSize() { return policiesMaxSize; } + @Override + public int getPolicyImportsLimit() { + return policyImportsLimit; + } + @Override public long getMessagesMaxSize() { return messagesMaxSize; @@ -92,11 +97,6 @@ public String getConfigPath() { return CONFIG_PATH; } - @Override - public int getPolicyImportsLimit() { - return policyImportsLimit; - } - @Override public boolean equals(@Nullable final Object o) { if (this == o) { diff --git a/base/service/src/main/java/org/eclipse/ditto/base/service/config/limits/LimitsConfig.java b/base/service/src/main/java/org/eclipse/ditto/base/service/config/limits/LimitsConfig.java index 0978ebecb09..94343f286c8 100644 --- a/base/service/src/main/java/org/eclipse/ditto/base/service/config/limits/LimitsConfig.java +++ b/base/service/src/main/java/org/eclipse/ditto/base/service/config/limits/LimitsConfig.java @@ -80,6 +80,11 @@ enum LimitsConfigValue implements KnownConfigValue { */ POLICIES_MAX_SIZE("policies.max-size", Constants.DEFAULT_ENTITY_MAX_SIZE), + /** + * The number of imports that a policy may contain. + */ + POLICY_IMPORTS_LIMIT("policies.imports-limit", 10), + /** * The maximum possible size of "Messages" entities in bytes. */ @@ -93,12 +98,7 @@ enum LimitsConfigValue implements KnownConfigValue { /** * The maximum pagination size to apply when searching for "Things" via "things-search". */ - THINGS_SEARCH_MAX_PAGE_SIZE(Constants.THINGS_SEARCH_PATH + "." + "max-page-size", 200), - - /** - * The number of imports that a policy may contain. - */ - POLICY_IMPORTS_LIMIT("imports-limit", 10); + THINGS_SEARCH_MAX_PAGE_SIZE(Constants.THINGS_SEARCH_PATH + "." + "max-page-size", 200); private final String path; private final Object defaultValue; diff --git a/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/DefaultLocalAskTimeoutConfig.java b/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/DefaultLocalAskTimeoutConfig.java new file mode 100644 index 00000000000..7df4ee807b8 --- /dev/null +++ b/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/DefaultLocalAskTimeoutConfig.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.ditto.base.service.config.supervision; + +import java.time.Duration; +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.internal.utils.config.ConfigWithFallback; +import org.eclipse.ditto.internal.utils.config.ScopedConfig; + +import com.typesafe.config.Config; + +/** + * This class is the default implementation of the local ACK timeout config. + */ +@Immutable +public class DefaultLocalAskTimeoutConfig implements LocalAskTimeoutConfig { + + private static final String CONFIG_PATH = "local-ask"; + private final Duration askTimeout; + + private DefaultLocalAskTimeoutConfig(final ScopedConfig config) { + askTimeout = config.getNonNegativeAndNonZeroDurationOrThrow(LocalAskTimeoutConfigValue.ASK_TIMEOUT); + } + + /** + * Returns an instance of {@code DefaultLocalAskTimeoutConfig} based on the settings of the specified Config. + * + * @param config is supposed to provide the settings of the local ASK timeout config at {@value #CONFIG_PATH}. + * @return the instance. + * @throws org.eclipse.ditto.internal.utils.config.DittoConfigError if {@code config} is invalid. + */ + public static DefaultLocalAskTimeoutConfig of(final Config config) { + return new DefaultLocalAskTimeoutConfig(ConfigWithFallback.newInstance(config, CONFIG_PATH, + LocalAskTimeoutConfig.LocalAskTimeoutConfigValue.values())); + } + + @Override + public Duration getLocalAckTimeout() { + return askTimeout; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DefaultLocalAskTimeoutConfig that = (DefaultLocalAskTimeoutConfig) o; + return Objects.equals(askTimeout, that.askTimeout); + } + + @Override + public int hashCode() { + return Objects.hash(askTimeout); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[" + + "askTimeout=" + askTimeout + + ']'; + } +} diff --git a/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/DefaultSupervisorConfig.java b/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/DefaultSupervisorConfig.java index 8f50cde1691..dae28521c12 100644 --- a/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/DefaultSupervisorConfig.java +++ b/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/DefaultSupervisorConfig.java @@ -30,9 +30,12 @@ public final class DefaultSupervisorConfig implements SupervisorConfig { private static final String CONFIG_PATH = "supervisor"; private final ExponentialBackOffConfig exponentialBackOffConfig; + private final DefaultLocalAskTimeoutConfig localAskTimeoutConfig; - private DefaultSupervisorConfig(final ExponentialBackOffConfig theExponentialBackOffConfig) { + private DefaultSupervisorConfig(final ExponentialBackOffConfig theExponentialBackOffConfig, + final DefaultLocalAskTimeoutConfig theLocalAskTimeoutConfig) { exponentialBackOffConfig = theExponentialBackOffConfig; + localAskTimeoutConfig = theLocalAskTimeoutConfig; } /** @@ -45,7 +48,8 @@ private DefaultSupervisorConfig(final ExponentialBackOffConfig theExponentialBac public static DefaultSupervisorConfig of(final Config config) { final ScopedConfig supervisorScopedConfig = DefaultScopedConfig.newInstance(config, CONFIG_PATH); - return new DefaultSupervisorConfig(DefaultExponentialBackOffConfig.of(supervisorScopedConfig)); + return new DefaultSupervisorConfig(DefaultExponentialBackOffConfig.of(supervisorScopedConfig), + DefaultLocalAskTimeoutConfig.of(supervisorScopedConfig)); } @Override @@ -53,6 +57,11 @@ public ExponentialBackOffConfig getExponentialBackOffConfig() { return exponentialBackOffConfig; } + @Override + public LocalAskTimeoutConfig getLocalAskTimeoutConfig() { + return localAskTimeoutConfig; + } + @Override public boolean equals(final Object o) { if (this == o) { diff --git a/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/LocalAskTimeoutConfig.java b/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/LocalAskTimeoutConfig.java new file mode 100644 index 00000000000..c14a1b3f3d5 --- /dev/null +++ b/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/LocalAskTimeoutConfig.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.ditto.base.service.config.supervision; + +import java.time.Duration; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.internal.utils.config.KnownConfigValue; + +/** + * Provides configuration settings for the local ACK timeout. + */ +@Immutable +public interface LocalAskTimeoutConfig { + + /** + * Timeout for local actor invocations - a small timeout should be more than sufficient as those are just method + * calls. + * @return the duration for a local ACK timeout calls. + */ + Duration getLocalAckTimeout(); + + + /** + * An enumeration of the known config path expressions and their associated default values for + * {@code LocalAskTimeoutConfigValue}. + */ + enum LocalAskTimeoutConfigValue implements KnownConfigValue { + + /** + * The local ACK timeout duration. + */ + ASK_TIMEOUT("timeout", Duration.ofSeconds(5L)); + + private final String path; + private final Duration defaultValue; + + LocalAskTimeoutConfigValue(final String thePath, final Duration theDefaultValue) { + + this.path = thePath; + this.defaultValue = theDefaultValue; + } + + @Override + public Object getDefaultValue() { + return defaultValue; + } + + @Override + public String getConfigPath() { + return path; + } + } +} diff --git a/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/SupervisorConfig.java b/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/SupervisorConfig.java index 9415b69a75f..10d8521a36d 100644 --- a/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/SupervisorConfig.java +++ b/base/service/src/main/java/org/eclipse/ditto/base/service/config/supervision/SupervisorConfig.java @@ -27,4 +27,9 @@ public interface SupervisorConfig { */ ExponentialBackOffConfig getExponentialBackOffConfig(); + /** Returns the config for supervisor local ACK timeout calls. + * @return the config. + */ + LocalAskTimeoutConfig getLocalAskTimeoutConfig(); + } diff --git a/base/service/src/test/java/org/eclipse/ditto/base/service/config/supervision/DefaultLocalAskTimeoutConfigTest.java b/base/service/src/test/java/org/eclipse/ditto/base/service/config/supervision/DefaultLocalAskTimeoutConfigTest.java new file mode 100644 index 00000000000..10edd813f2a --- /dev/null +++ b/base/service/src/test/java/org/eclipse/ditto/base/service/config/supervision/DefaultLocalAskTimeoutConfigTest.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.ditto.base.service.config.supervision; + +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.time.Duration; + +import org.assertj.core.api.JUnitSoftAssertions; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Unit test for {@link DefaultLocalAskTimeoutConfigTest}. + */ +public class DefaultLocalAskTimeoutConfigTest { + + private static Config supervisorLocalAskTimeoutConfig; + + @Rule + public final JUnitSoftAssertions softly = new JUnitSoftAssertions(); + + @BeforeClass + public static void initTestFixture() { + supervisorLocalAskTimeoutConfig = ConfigFactory.load("local-ask-timout-test"); + } + + @Test + public void assertImmutability() { + assertInstancesOf(DefaultLocalAskTimeoutConfig.class, + areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(DefaultLocalAskTimeoutConfig.class) + .usingGetClass() + .verify(); + } + + @Test + public void underTestReturnsDefaultValuesIfBaseConfigWasEmpty() { + final DefaultLocalAskTimeoutConfig underTest = DefaultLocalAskTimeoutConfig.of(ConfigFactory.empty()); + + softly.assertThat(underTest.getLocalAckTimeout()) + .as(LocalAskTimeoutConfig.LocalAskTimeoutConfigValue.ASK_TIMEOUT.getConfigPath()) + .isEqualTo(LocalAskTimeoutConfig.LocalAskTimeoutConfigValue.ASK_TIMEOUT.getDefaultValue()); + } + + @Test + public void underTestReturnsValuesOfConfigFile() { + final DefaultLocalAskTimeoutConfig underTest = DefaultLocalAskTimeoutConfig.of(supervisorLocalAskTimeoutConfig); + + softly.assertThat(underTest.getLocalAckTimeout()) + .as(LocalAskTimeoutConfig.LocalAskTimeoutConfigValue.ASK_TIMEOUT.getConfigPath()) + .isEqualTo(Duration.ofSeconds(10L)); + } +} \ No newline at end of file diff --git a/base/service/src/test/resources/local-ask-timout-test.conf b/base/service/src/test/resources/local-ask-timout-test.conf new file mode 100644 index 00000000000..110938ddc6f --- /dev/null +++ b/base/service/src/test/resources/local-ask-timout-test.conf @@ -0,0 +1,3 @@ +local-ask { + timeout = 10s +} diff --git a/bom/pom.xml b/bom/pom.xml index d7ff2b70b1b..6b59efa521b 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -34,7 +34,7 @@ 0.9.5 - 2.13.4.20221013 + 2.14.3 1.4.2 0.6.1 2.6.20 @@ -85,7 +85,7 @@ --> 3.2.0 - 3.1.1 + 3.1.6 3.11 diff --git a/connectivity/model/pom.xml b/connectivity/model/pom.xml index 9ac624708b7..01801773765 100644 --- a/connectivity/model/pom.xml +++ b/connectivity/model/pom.xml @@ -124,7 +124,7 @@ - + org.eclipse.ditto.connectivity.model.Connection#toJson(org.eclipse.ditto.base.model.json.JsonSchemaVersion,org.eclipse.ditto.json.JsonFieldSelector) diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnection.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnection.java index eea09317d52..bb8dc0f8e31 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnection.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnection.java @@ -15,6 +15,8 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.text.MessageFormat; +import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -31,6 +33,8 @@ import javax.annotation.Nullable; +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.json.JsonArray; import org.eclipse.ditto.json.JsonCollectors; @@ -56,6 +60,9 @@ abstract class AbstractConnection implements Connection { @Nullable private final Credentials credentials; @Nullable private final String trustedCertificates; @Nullable private final ConnectionLifecycle lifecycle; + @Nullable private final ConnectionRevision revision; + @Nullable private final Instant modified; + @Nullable private final Instant created; private final List sources; private final List targets; private final int clientCount; @@ -85,12 +92,15 @@ abstract class AbstractConnection implements Connection { payloadMappingDefinition = builder.payloadMappingDefinition; tags = Collections.unmodifiableSet(new LinkedHashSet<>(builder.tags)); lifecycle = builder.lifecycle; + revision = builder.revision; + modified = builder.modified; + created = builder.created; sshTunnel = builder.sshTunnel; } abstract ConnectionUri getConnectionUri(@Nullable String builderConnectionUri); - static void buildFromJson (final JsonObject jsonObject, final AbstractConnectionBuilder builder) { + static void buildFromJson(final JsonObject jsonObject, final AbstractConnectionBuilder builder) { final MappingContext mappingContext = jsonObject.getValue(JsonFields.MAPPING_CONTEXT) .map(ConnectivityModelFactory::mappingContextFromJson) .orElse(null); @@ -112,13 +122,22 @@ static void buildFromJson (final JsonObject jsonObject, final AbstractConnection jsonObject.getValue(JsonFields.LIFECYCLE) .flatMap(ConnectionLifecycle::forName).ifPresent(builder::lifecycle); - jsonObject.getValue(JsonFields.CREDENTIALS).ifPresent(builder::credentialsFromJson); + jsonObject.getValue(JsonFields.REVISION) + .map(ConnectionRevision::newInstance).ifPresent(builder::revision); + jsonObject.getValue(JsonFields.MODIFIED) + .map(AbstractConnection::tryToParseInstant).ifPresent(builder::modified); + jsonObject.getValue(JsonFields.CREATED) + .map(AbstractConnection::tryToParseInstant).ifPresent(builder::created); + jsonObject.getValue(JsonFields.CREDENTIALS) + .filter(f -> !f.isNull()) + .ifPresent(builder::credentialsFromJson); jsonObject.getValue(JsonFields.CLIENT_COUNT).ifPresent(builder::clientCount); jsonObject.getValue(JsonFields.FAILOVER_ENABLED).ifPresent(builder::failoverEnabled); jsonObject.getValue(JsonFields.VALIDATE_CERTIFICATES).ifPresent(builder::validateCertificate); jsonObject.getValue(JsonFields.PROCESSOR_POOL_SIZE).ifPresent(builder::processorPoolSize); jsonObject.getValue(JsonFields.TRUSTED_CERTIFICATES).ifPresent(builder::trustedCertificates); jsonObject.getValue(JsonFields.SSH_TUNNEL) + .filter(f -> !f.isNull()) .ifPresent(jsonFields -> builder.sshTunnel(ImmutableSshTunnel.fromJson(jsonFields))); } @@ -159,6 +178,7 @@ private static List getTargets(final JsonObject jsonObject) { private static Map getSpecificConfiguration(final JsonObject jsonObject) { return jsonObject.getValue(JsonFields.SPECIFIC_CONFIG) + .filter(f -> !f.isNull()) .filter(JsonValue::isObject) .map(JsonValue::asObject) .map(JsonObject::stream) @@ -169,6 +189,7 @@ private static Map getSpecificConfiguration(final JsonObject jso private static Set getTags(final JsonObject jsonObject) { return jsonObject.getValue(JsonFields.TAGS) + .filter(f -> !f.isNull()) .map(array -> array.stream() .filter(JsonValue::isString) .map(JsonValue::asString) @@ -296,6 +317,36 @@ public Optional getLifecycle() { return Optional.ofNullable(lifecycle); } + @Override + public Optional getEntityId() { + return Optional.of(id); + } + + @Override + public Optional getRevision() { + return Optional.ofNullable(revision); + } + + @Override + public Optional getModified() { + return Optional.ofNullable(modified); + } + + @Override + public Optional getCreated() { + return Optional.ofNullable(created); + } + + @Override + public Optional getMetadata() { + return Optional.empty(); // currently not metadata support for connections + } + + @Override + public boolean isDeleted() { + return ConnectionLifecycle.DELETED.equals(lifecycle); + } + static ConnectionBuilder fromConnection(final Connection connection, final AbstractConnectionBuilder builder) { checkNotNull(connection, "Connection"); @@ -316,7 +367,10 @@ static ConnectionBuilder fromConnection(final Connection connection, final Abstr .name(connection.getName().orElse(null)) .sshTunnel(connection.getSshTunnel().orElse(null)) .tags(connection.getTags()) - .lifecycle(connection.getLifecycle().orElse(null)); + .lifecycle(connection.getLifecycle().orElse(null)) + .revision(connection.getRevision().orElse(null)) + .modified(connection.getModified().orElse(null)) + .created(connection.getCreated().orElse(null)); } @Override @@ -329,6 +383,15 @@ public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate< } jsonObjectBuilder.set(JsonFields.ID, String.valueOf(id), predicate); jsonObjectBuilder.set(JsonFields.NAME, name, predicate); + if (null != revision) { + jsonObjectBuilder.set(JsonFields.REVISION, revision.toLong(), predicate); + } + if (null != modified) { + jsonObjectBuilder.set(JsonFields.MODIFIED, modified.toString(), predicate); + } + if (null != created) { + jsonObjectBuilder.set(JsonFields.CREATED, created.toString(), predicate); + } jsonObjectBuilder.set(JsonFields.CONNECTION_TYPE, connectionType.getName(), predicate); jsonObjectBuilder.set(JsonFields.CONNECTION_STATUS, connectionStatus.getName(), predicate); jsonObjectBuilder.set(JsonFields.URI, uri.toString(), predicate); @@ -368,6 +431,15 @@ public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate< return jsonObjectBuilder.build(); } + private static Instant tryToParseInstant(final CharSequence dateTime) { + try { + return Instant.parse(dateTime); + } catch (final DateTimeParseException e) { + throw new JsonParseException("The JSON object's field '" + Connection.JsonFields.MODIFIED.getPointer() + "' " + + "is not in ISO-8601 format as expected"); + } + } + @SuppressWarnings("OverlyComplexMethod") @Override public boolean equals(@Nullable final Object o) { @@ -394,6 +466,9 @@ public boolean equals(@Nullable final Object o) { Objects.equals(specificConfig, that.specificConfig) && Objects.equals(payloadMappingDefinition, that.payloadMappingDefinition) && Objects.equals(lifecycle, that.lifecycle) && + Objects.equals(revision, that.revision) && + Objects.equals(modified, that.modified) && + Objects.equals(created, that.created) && Objects.equals(sshTunnel, that.sshTunnel) && Objects.equals(tags, that.tags); } @@ -402,7 +477,7 @@ public boolean equals(@Nullable final Object o) { public int hashCode() { return Objects.hash(id, name, connectionType, connectionStatus, sources, targets, clientCount, failOverEnabled, credentials, trustedCertificates, uri, validateCertificate, processorPoolSize, specificConfig, - payloadMappingDefinition, sshTunnel, tags, lifecycle); + payloadMappingDefinition, sshTunnel, tags, lifecycle, revision, modified, created); } @Override @@ -426,6 +501,9 @@ public String toString() { ", payloadMappingDefinition=" + payloadMappingDefinition + ", tags=" + tags + ", lifecycle=" + lifecycle + + ", revision=" + revision + + ", modified=" + modified + + ", created=" + created + "]"; } diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnectionBuilder.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnectionBuilder.java index 5bf00d4797a..6243c22352f 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnectionBuilder.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnectionBuilder.java @@ -16,6 +16,7 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.text.MessageFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -49,6 +50,9 @@ abstract class AbstractConnectionBuilder implements ConnectionBuilder { @Nullable MappingContext mappingContext = null; @Nullable String trustedCertificates; @Nullable ConnectionLifecycle lifecycle = null; + @Nullable ConnectionRevision revision = null; + @Nullable Instant modified = null; + @Nullable Instant created = null; @Nullable SshTunnel sshTunnel = null; // optional with default: @@ -192,6 +196,24 @@ public ConnectionBuilder lifecycle(@Nullable final ConnectionLifecycle lifecycle return this; } + @Override + public ConnectionBuilder revision(@Nullable final ConnectionRevision revision) { + this.revision = revision; + return this; + } + + @Override + public ConnectionBuilder modified(@Nullable final Instant modified) { + this.modified = modified; + return this; + } + + @Override + public ConnectionBuilder created(@Nullable final Instant created) { + this.created = created; + return this; + } + @Override public ConnectionBuilder sshTunnel(@Nullable final SshTunnel sshTunnel) { this.sshTunnel = sshTunnel; diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/Connection.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/Connection.java index b9a99b60f1e..bcb51110a54 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/Connection.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/Connection.java @@ -20,21 +20,19 @@ import javax.annotation.concurrent.Immutable; +import org.eclipse.ditto.base.model.entity.Entity; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.base.model.json.Jsonifiable; import org.eclipse.ditto.json.JsonArray; import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonField; import org.eclipse.ditto.json.JsonFieldDefinition; -import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; /** * Represents a connection within the Connectivity service. */ @Immutable -public interface Connection extends Jsonifiable.WithFieldSelectorAndPredicate { +public interface Connection extends Entity { /** * Returns the identifier of this {@code Connection}. @@ -242,11 +240,6 @@ default JsonObject toJson() { return toJson(FieldType.notHidden()); } - @Override - default JsonObject toJson(final JsonSchemaVersion schemaVersion, final JsonFieldSelector fieldSelector) { - return toJson(schemaVersion, FieldType.notHidden()).get(fieldSelector); - } - /** * An enumeration of the known {@code JsonField}s of a {@code Connection}. */ @@ -261,6 +254,33 @@ final class JsonFields { FieldType.HIDDEN, JsonSchemaVersion.V_2); + /** + * JSON field containing the Connection's revision. + * @since 3.2.0 + */ + public static final JsonFieldDefinition REVISION = JsonFactory.newLongFieldDefinition("_revision", + FieldType.SPECIAL, + FieldType.HIDDEN, + JsonSchemaVersion.V_2); + + /** + * JSON field containing the Connection's modified timestamp in ISO-8601 format. + * @since 3.2.0 + */ + public static final JsonFieldDefinition MODIFIED = JsonFactory.newStringFieldDefinition("_modified", + FieldType.SPECIAL, + FieldType.HIDDEN, + JsonSchemaVersion.V_2); + + /** + * JSON field containing the Connection's created timestamp in ISO-8601 format. + * @since 3.2.0 + */ + public static final JsonFieldDefinition CREATED = JsonFactory.newStringFieldDefinition("_created", + FieldType.SPECIAL, + FieldType.HIDDEN, + JsonSchemaVersion.V_2); + /** * JSON field containing the {@code Connection} identifier. */ diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionBuilder.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionBuilder.java index a124f64ccaa..56daf691b7b 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionBuilder.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionBuilder.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.connectivity.model; +import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; @@ -203,6 +204,44 @@ default ConnectionBuilder credentialsFromJson(final JsonObject jsonObject) { */ ConnectionBuilder lifecycle(@Nullable ConnectionLifecycle lifecycle); + /** + * Sets the given revision number to this builder. + * + * @param revisionNumber the revision number to be set. + * @return this builder to allow method chaining. + * @since 3.2.0 + */ + default ConnectionBuilder revision(final long revisionNumber) { + return revision(ConnectionRevision.newInstance(revisionNumber)); + } + + /** + * Sets the {@link ConnectionRevision} of the connection. + * + * @param revision the connection revision + * @return this builder + * @since 3.2.0 + */ + ConnectionBuilder revision(@Nullable ConnectionRevision revision); + + /** + * Sets the given modified timestamp to this builder. + * + * @param modified the timestamp to be set. + * @return this builder to allow method chaining. + * @since 3.2.0 + */ + ConnectionBuilder modified(@Nullable Instant modified); + + /** + * Sets the given created timestamp to this builder. + * + * @param created the created timestamp to be set. + * @return this builder to allow method chaining. + * @since 3.2.0 + */ + ConnectionBuilder created(@Nullable Instant created); + /** * Sets the {@link SshTunnel} of the connection. * diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionRevision.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionRevision.java new file mode 100755 index 00000000000..2986f7a82d3 --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionRevision.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model; + +import org.eclipse.ditto.base.model.entity.Revision; + +/** + * Represents the current revision of a Connection. + */ +public interface ConnectionRevision extends Revision { + + /** + * Returns a new immutable {@code ConnectionRevision} which is initialised with the given revision number. + * + * @param revisionNumber the {@code long} value of the revision. + * @return the new immutable {@code ConnectionRevision}. + */ + static ConnectionRevision newInstance(final long revisionNumber) { + return ConnectivityModelFactory.newConnectionRevision(revisionNumber); + } + +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectivityModelFactory.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectivityModelFactory.java index 61ca499af91..4ace7c5435f 100755 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectivityModelFactory.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectivityModelFactory.java @@ -991,4 +991,15 @@ public static LogEntryBuilder newLogEntryBuilder(final String correlationId, return ImmutableLogEntry.getBuilder(correlationId, timestamp, logCategory, logType, logLevel, message); } + /** + * Returns a new immutable {@link ConnectionRevision} which is initialised with the given revision number. + * + * @param revisionNumber the {@code long} value of the revision. + * @return the new immutable {@code ConnectionRevision}. + * @since 3.2.0 + */ + public static ConnectionRevision newConnectionRevision(final long revisionNumber) { + return ImmutableConnectionRevision.of(revisionNumber); + } + } diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableConnectionRevision.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableConnectionRevision.java new file mode 100755 index 00000000000..c9ec22b8000 --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableConnectionRevision.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +/** + * An immutable implementation of {@link ConnectionRevision}. + */ +@Immutable +final class ImmutableConnectionRevision implements ConnectionRevision { + + private final long value; + + private ImmutableConnectionRevision(final long theValue) { + value = theValue; + } + + /** + * Returns a new instance of {@code ConnectionRevision} with the given value. + * + * @param value the value of the new revision. + * @return a new Connection revision. + */ + public static ImmutableConnectionRevision of(final long value) { + return new ImmutableConnectionRevision(value); + } + + @Override + public boolean isGreaterThan(final ConnectionRevision other) { + return 0 < compareTo(other); + } + + @Override + public boolean isGreaterThanOrEqualTo(final ConnectionRevision other) { + return 0 <= compareTo(other); + } + + @Override + public boolean isLowerThan(final ConnectionRevision other) { + return 0 > compareTo(other); + } + + @Override + public boolean isLowerThanOrEqualTo(final ConnectionRevision other) { + return 0 >= compareTo(other); + } + + @Override + public ConnectionRevision increment() { + return of(value + 1); + } + + @Override + public long toLong() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final ImmutableConnectionRevision that = (ImmutableConnectionRevision) o; + return value == that.value; + } + + @Override + public int hashCode() { + return Objects.hash(value); + } + + @Override + public int compareTo(final ConnectionRevision o) { + checkNotNull(o, "other revision to compare this revision with"); + return Long.compare(value, o.toLong()); + } + +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableMappingContext.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableMappingContext.java index 286063a76d9..5cac0bef77e 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableMappingContext.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableMappingContext.java @@ -25,13 +25,13 @@ import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.NotThreadSafe; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.json.JsonCollectors; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonField; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.json.JsonValue; -import org.eclipse.ditto.base.model.json.JsonSchemaVersion; /** * Immutable implementation of {@link MappingContext}. @@ -69,14 +69,18 @@ public static MappingContext fromJson(final JsonObject jsonObject) { final Builder builder = new Builder(mappingEngine, options); builder.incomingConditions( - jsonObject.getValue(JsonFields.INCOMING_CONDITIONS).orElse(JsonObject.empty()).stream() + jsonObject.getValue(JsonFields.INCOMING_CONDITIONS) + .filter(f -> !f.isNull()) + .orElse(JsonObject.empty()).stream() .collect(Collectors.toMap( e -> e.getKey().toString(), e -> e.getValue().isString() ? e.getValue().asString() : e.getValue().toString()) )); builder.outgoingConditions( - jsonObject.getValue(JsonFields.OUTGOING_CONDITIONS).orElse(JsonObject.empty()).stream() + jsonObject.getValue(JsonFields.OUTGOING_CONDITIONS) + .filter(f -> !f.isNull()) + .orElse(JsonObject.empty()).stream() .collect(Collectors.toMap( e -> e.getKey().toString(), e -> e.getValue().isString() ? e.getValue().asString() : e.getValue().toString()) diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableReplyTarget.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableReplyTarget.java index a8ac462faad..e8c009505c7 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableReplyTarget.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableReplyTarget.java @@ -121,9 +121,11 @@ static Optional fromJsonOptional(final JsonObject jsonObject) { return jsonObject.getValue(JsonFields.ADDRESS).map(address -> new Builder() .address(address) .headerMapping(jsonObject.getValue(JsonFields.HEADER_MAPPING) + .filter(f -> !f.isNull()) .map(ConnectivityModelFactory::newHeaderMapping) .orElse(null)) .expectedResponseTypes(jsonObject.getValue(JsonFields.EXPECTED_RESPONSE_TYPES) + .filter(f -> !f.isNull()) .map(jsonArray -> jsonArray.stream() .map(JsonValue::asString) .map(ResponseType::fromName) diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableSource.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableSource.java index 1ef25b6783c..b1b3ef15d1c 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableSource.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableSource.java @@ -253,18 +253,26 @@ public static Source fromJson(final JsonObject jsonObject, final int index) { authorizationSubjects); final Enforcement readEnforcement = - jsonObject.getValue(JsonFields.ENFORCEMENT).map(ImmutableEnforcement::fromJson).orElse(null); + jsonObject.getValue(JsonFields.ENFORCEMENT) + .filter(f -> !f.isNull()) + .map(ImmutableEnforcement::fromJson) + .orElse(null); final FilteredAcknowledgementRequest readAcknowledgementRequests = jsonObject.getValue(JsonFields.ACKNOWLEDGEMENT_REQUESTS) + .filter(f -> !f.isNull()) .map(FilteredAcknowledgementRequest::fromJson) .orElse(null); final HeaderMapping readHeaderMapping = - jsonObject.getValue(JsonFields.HEADER_MAPPING).map(ImmutableHeaderMapping::fromJson).orElse(null); + jsonObject.getValue(JsonFields.HEADER_MAPPING) + .filter(f -> !f.isNull()) + .map(ImmutableHeaderMapping::fromJson) + .orElse(null); final PayloadMapping readPayloadMapping = jsonObject.getValue(JsonFields.PAYLOAD_MAPPING) + .filter(f -> !f.isNull()) .map(ImmutablePayloadMapping::fromJson) .orElse(ConnectivityModelFactory.emptyPayloadMapping()); @@ -273,6 +281,7 @@ public static Source fromJson(final JsonObject jsonObject, final int index) { final ReplyTarget readReplyTarget = jsonObject.getValue(JsonFields.REPLY_TARGET) + .filter(f -> !f.isNull()) .flatMap(ImmutableReplyTarget::fromJsonOptional) .orElse(null); diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableTarget.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableTarget.java index e81329e7a43..6f3be41d147 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableTarget.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableTarget.java @@ -193,11 +193,13 @@ public static Target fromJson(final JsonObject jsonObject) { final HeaderMapping readHeaderMapping = jsonObject.getValue(JsonFields.HEADER_MAPPING) + .filter(f -> !f.isNull()) .map(ImmutableHeaderMapping::fromJson) .orElse(null); final PayloadMapping readMapping = jsonObject.getValue(JsonFields.PAYLOAD_MAPPING) + .filter(f -> !f.isNull()) .map(ImmutablePayloadMapping::fromJson) .orElse(ConnectivityModelFactory.emptyPayloadMapping()); diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java index 6c33196e410..53ead396749 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java @@ -12,14 +12,16 @@ */ package org.eclipse.ditto.connectivity.model.signals.commands; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; -import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.entity.type.WithEntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonPointer; /** * Base interface for all commands which are understood by the Connectivity service. Implementations of this interface @@ -27,7 +29,8 @@ * * @param the type of the implementing class. */ -public interface ConnectivityCommand> extends Command { +public interface ConnectivityCommand> extends Command, + WithEntityType { /** * Type Prefix of Connectivity commands. @@ -57,6 +60,17 @@ default String getTypePrefix() { @Override T setDittoHeaders(DittoHeaders dittoHeaders); + /** + * Returns the entity type {@link ConnectivityConstants#ENTITY_TYPE}. + * + * @return the Connection entity type. + * @since 3.2.0 + */ + @Override + default EntityType getEntityType() { + return ConnectivityConstants.ENTITY_TYPE; + } + /** * This class contains definitions for all specific fields of a {@code ConnectivityCommand}'s JSON representation. */ diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionHistoryNotAccessibleException.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionHistoryNotAccessibleException.java new file mode 100755 index 00000000000..e51a650b3ae --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionHistoryNotAccessibleException.java @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model.signals.commands.exceptions; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.net.URI; +import java.text.MessageFormat; +import java.time.Instant; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.ConnectivityException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Thrown if historical data of the Connection was either not present in Ditto at all or if the requester had insufficient + * permissions to access it. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableException(errorCode = ConnectionHistoryNotAccessibleException.ERROR_CODE) +public final class ConnectionHistoryNotAccessibleException extends DittoRuntimeException + implements ConnectivityException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "connection.history.notfound"; + + private static final String MESSAGE_TEMPLATE = + "The Connection with ID ''{0}'' at revision ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String MESSAGE_TEMPLATE_TS = + "The Connection with ID ''{0}'' at timestamp ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String DEFAULT_DESCRIPTION = + "Check if the ID of your requested Connection was correct, you have sufficient permissions and ensure that the " + + "asked for revision/timestamp does not exceed the history-retention-duration."; + + private static final long serialVersionUID = -998877665544332221L; + + private ConnectionHistoryNotAccessibleException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + super(ERROR_CODE, HttpStatus.NOT_FOUND, dittoHeaders, message, description, cause, href); + } + + private static String getMessage(final ConnectionId connectionId, final long revision) { + checkNotNull(connectionId, "connectionId"); + return MessageFormat.format(MESSAGE_TEMPLATE, String.valueOf(connectionId), String.valueOf(revision)); + } + + private static String getMessage(final ConnectionId connectionId, final Instant timestamp) { + checkNotNull(connectionId, "connectionId"); + checkNotNull(timestamp, "timestamp"); + return MessageFormat.format(MESSAGE_TEMPLATE_TS, String.valueOf(connectionId), timestamp.toString()); + } + + /** + * A mutable builder for a {@code ConnectionHistoryNotAccessibleException}. + * + * @param connectionId the ID of the connection. + * @param revision the asked for revision of the connection. + * @return the builder. + * @throws NullPointerException if {@code connectionId} is {@code null}. + */ + public static Builder newBuilder(final ConnectionId connectionId, final long revision) { + return new Builder(connectionId, revision); + } + + /** + * A mutable builder for a {@code ConnectionHistoryNotAccessibleException}. + * + * @param connectionId the ID of the connection. + * @param timestamp the asked for timestamp of the connection. + * @return the builder. + * @throws NullPointerException if {@code connectionId} is {@code null}. + */ + public static Builder newBuilder(final ConnectionId connectionId, final Instant timestamp) { + return new Builder(connectionId, timestamp); + } + + /** + * Constructs a new {@code ConnectionHistoryNotAccessibleException} object with given message. + * + * @param message detail message. This message can be later retrieved by the {@link #getMessage()} method. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new ConnectionHistoryNotAccessibleException. + * @throws NullPointerException if {@code dittoHeaders} is {@code null}. + */ + public static ConnectionHistoryNotAccessibleException fromMessage(@Nullable final String message, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromMessage(message, dittoHeaders, new Builder()); + } + + /** + * Constructs a new {@code ConnectionHistoryNotAccessibleException} object with the exception message extracted from the given + * JSON object. + * + * @param jsonObject the JSON to read the {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new ConnectionHistoryNotAccessibleException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static ConnectionHistoryNotAccessibleException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link ConnectionHistoryNotAccessibleException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() { + description(DEFAULT_DESCRIPTION); + } + + private Builder(final ConnectionId connectionId, final long revision) { + this(); + message(ConnectionHistoryNotAccessibleException.getMessage(connectionId, revision)); + } + + private Builder(final ConnectionId connectionId, final Instant timestamp) { + this(); + message(ConnectionHistoryNotAccessibleException.getMessage(connectionId, timestamp)); + } + + @Override + protected ConnectionHistoryNotAccessibleException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new ConnectionHistoryNotAccessibleException(dittoHeaders, message, description, cause, href); + } + + } + +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionFailedException.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionFailedException.java new file mode 100644 index 00000000000..6746f57942c --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionFailedException.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model.signals.commands.exceptions; + +import java.net.URI; +import java.text.MessageFormat; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.connectivity.model.ConnectivityException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Thrown when validating a precondition header fails on a Connection or one of its sub-entities. + * @since 3.2.0 + */ +@Immutable +@JsonParsableException(errorCode = ConnectionPreconditionFailedException.ERROR_CODE) +public final class ConnectionPreconditionFailedException extends DittoRuntimeException implements + ConnectivityException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "precondition.failed"; + + private static final String MESSAGE_TEMPLATE = + "The comparison of precondition header ''{0}'' for the requested connection resource evaluated to false." + + " Header value: ''{1}'', actual entity-tag: ''{2}''."; + + private static final String DEFAULT_DESCRIPTION = "The comparison of the provided precondition header with the " + + "current ETag value of the requested connection resource evaluated to false. Check the value of your " + + "conditional header value."; + + private ConnectionPreconditionFailedException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + super(ERROR_CODE, HttpStatus.PRECONDITION_FAILED, dittoHeaders, message, description, cause, href); + } + + /** + * A mutable builder for a {@link ConnectionPreconditionFailedException}. + * + * @param conditionalHeaderName the name of the conditional header. + * @param expected the expected value. + * @param actual the actual ETag value. + * @return the builder. + */ + public static Builder newBuilder(final String conditionalHeaderName, final String expected, final String actual) { + return new Builder(conditionalHeaderName, expected, actual); + } + + /** + * Constructs a new {@link ConnectionPreconditionFailedException} object with the exception message extracted from + * the given JSON object. + * + * @param jsonObject the JSON to read the {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new {@link ConnectionPreconditionFailedException}. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static ConnectionPreconditionFailedException fromJson(final JsonObject jsonObject, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link ConnectionPreconditionFailedException}. + */ + @NotThreadSafe + public static final class Builder + extends DittoRuntimeExceptionBuilder { + + private Builder() { + description(DEFAULT_DESCRIPTION); + } + + private Builder(final String conditionalHeaderName, final String expected, final String actual) { + this(); + message(MessageFormat.format(MESSAGE_TEMPLATE, conditionalHeaderName, expected, actual)); + } + + @Override + protected ConnectionPreconditionFailedException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new ConnectionPreconditionFailedException(dittoHeaders, message, description, cause, href); + } + + } +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionNotModifiedException.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionNotModifiedException.java new file mode 100644 index 00000000000..dfaeaf5e828 --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionNotModifiedException.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model.signals.commands.exceptions; + +import java.net.URI; +import java.text.MessageFormat; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.connectivity.model.ConnectivityException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Thrown when validating a precondition header on a connection or one of its sub-entities leads to status + * {@link org.eclipse.ditto.base.model.common.HttpStatus#NOT_MODIFIED}. + */ +@Immutable +@JsonParsableException(errorCode = ConnectionPreconditionNotModifiedException.ERROR_CODE) +public final class ConnectionPreconditionNotModifiedException extends DittoRuntimeException + implements ConnectivityException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "precondition.notmodified"; + + private static final String MESSAGE_TEMPLATE = + "The comparison of precondition header ''if-none-match'' for the requested connection resource evaluated to " + + "false. Expected: ''{0}'' not to match actual: ''{1}''."; + + private static final String DEFAULT_DESCRIPTION = + "The comparison of the provided precondition header ''if-none-match'' with the current ETag value of the " + + "requested connection resource evaluated to false. Check the value of your conditional header value."; + + private ConnectionPreconditionNotModifiedException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + super(ERROR_CODE, HttpStatus.NOT_MODIFIED, dittoHeaders, message, description, cause, href); + } + + /** + * A mutable builder for a {@link ConnectionPreconditionNotModifiedException}. + * + * @return the builder. + * @since 3.3.0 + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * A mutable builder for a {@link ConnectionPreconditionNotModifiedException}. + * + * @param expectedNotToMatch the value which was expected not to match {@code matched} value. + * @param matched the matched value. + * @return the builder. + */ + public static Builder newBuilder(final String expectedNotToMatch, final String matched) { + return new Builder(expectedNotToMatch, matched); + } + + /** + * Constructs a new {@link ConnectionPreconditionNotModifiedException} object with the exception message extracted from + * the given JSON object. + * + * @param jsonObject the JSON to read the + * {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new ConditionalHeadersNotModifiedException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static ConnectionPreconditionNotModifiedException fromJson(final JsonObject jsonObject, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link ConnectionPreconditionNotModifiedException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() { + description(DEFAULT_DESCRIPTION); + } + + private Builder(final String expectedNotToMatch, final String matched) { + this(); + message(MessageFormat.format(MESSAGE_TEMPLATE, expectedNotToMatch, matched)); + } + + @Override + protected ConnectionPreconditionNotModifiedException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new ConnectionPreconditionNotModifiedException(dittoHeaders, message, description, cause, href); + } + + } +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnection.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnection.java index bd9e0c25df1..dfafac737c3 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnection.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnection.java @@ -15,24 +15,29 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.util.Objects; +import java.util.Optional; import java.util.function.Predicate; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; -import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonField; -import org.eclipse.ditto.json.JsonObject; -import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonParsableCommand; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.connectivity.model.ConnectionId; -import org.eclipse.ditto.connectivity.model.WithConnectionId; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.AbstractCommand; import org.eclipse.ditto.base.model.signals.commands.CommandJsonDeserializer; +import org.eclipse.ditto.base.model.signals.commands.WithSelectedFields; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.WithConnectionId; +import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonFieldSelector; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; /** * Command which retrieves a {@link org.eclipse.ditto.connectivity.model.Connection}. @@ -40,7 +45,8 @@ @Immutable @JsonParsableCommand(typePrefix = ConnectivityCommand.TYPE_PREFIX, name = RetrieveConnection.NAME) public final class RetrieveConnection extends AbstractCommand - implements ConnectivityQueryCommand, WithConnectionId, SignalWithEntityId { + implements ConnectivityQueryCommand, WithConnectionId, WithSelectedFields, + SignalWithEntityId { /** * Name of this command. @@ -52,11 +58,19 @@ public final class RetrieveConnection extends AbstractCommand JSON_SELECTED_FIELDS = + JsonFactory.newStringFieldDefinition("selectedFields", FieldType.REGULAR, + JsonSchemaVersion.V_2); + private final ConnectionId connectionId; + @Nullable private final JsonFieldSelector selectedFields; - private RetrieveConnection(final ConnectionId connectionId, final DittoHeaders dittoHeaders) { + private RetrieveConnection(final ConnectionId connectionId, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { super(TYPE, dittoHeaders); this.connectionId = connectionId; + this.selectedFields = selectedFields; } /** @@ -69,7 +83,23 @@ private RetrieveConnection(final ConnectionId connectionId, final DittoHeaders d */ public static RetrieveConnection of(final ConnectionId connectionId, final DittoHeaders dittoHeaders) { checkNotNull(connectionId, "Connection ID"); - return new RetrieveConnection(connectionId, dittoHeaders); + return new RetrieveConnection(connectionId, null, dittoHeaders); + } + + /** + * Returns a new instance of {@code RetrieveConnection}. + * + * @param connectionId the identifier of the connection to be retrieved. + * @param selectedFields the fields of the JSON representation of the Connection to retrieve. + * @param dittoHeaders the headers of the request. + * @return a new RetrieveConnection command. + * @throws NullPointerException if any argument is {@code null}. + */ + public static RetrieveConnection of(final ConnectionId connectionId, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { + checkNotNull(connectionId, "Connection ID"); + return new RetrieveConnection(connectionId, selectedFields, dittoHeaders); } /** @@ -101,8 +131,12 @@ public static RetrieveConnection fromJson(final JsonObject jsonObject, final Dit return new CommandJsonDeserializer(TYPE, jsonObject).deserialize(() -> { final String readConnectionId = jsonObject.getValueOrThrow(ConnectivityCommand.JsonFields.JSON_CONNECTION_ID); final ConnectionId connectionId = ConnectionId.of(readConnectionId); + final Optional selectedFields = jsonObject.getValue(JSON_SELECTED_FIELDS) + .map(str -> JsonFactory.newFieldSelector(str, JsonFactory.newParseOptionsBuilder() + .withoutUrlDecoding() + .build())); - return of(connectionId, dittoHeaders); + return of(connectionId, selectedFields.orElse(null), dittoHeaders); }); } @@ -113,6 +147,9 @@ protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final Js final Predicate predicate = schemaVersion.and(thePredicate); jsonObjectBuilder.set(ConnectivityCommand.JsonFields.JSON_CONNECTION_ID, String.valueOf(connectionId), predicate); + if (null != selectedFields) { + jsonObjectBuilder.set(JSON_SELECTED_FIELDS, selectedFields.toString(), predicate); + } } @Override @@ -125,9 +162,14 @@ public Category getCategory() { return Category.QUERY; } + @Override + public Optional getSelectedFields() { + return Optional.ofNullable(selectedFields); + } + @Override public RetrieveConnection setDittoHeaders(final DittoHeaders dittoHeaders) { - return of(connectionId, dittoHeaders); + return of(connectionId, selectedFields, dittoHeaders); } @Override @@ -147,12 +189,13 @@ public boolean equals(@Nullable final Object o) { return false; } final RetrieveConnection that = (RetrieveConnection) o; - return Objects.equals(connectionId, that.connectionId); + return Objects.equals(connectionId, that.connectionId) && + Objects.equals(selectedFields, that.selectedFields); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), connectionId); + return Objects.hash(super.hashCode(), connectionId, selectedFields); } @Override @@ -160,6 +203,7 @@ public String toString() { return getClass().getSimpleName() + " [" + super.toString() + ", connectionId=" + connectionId + + ", selectedFields=" + selectedFields + "]"; } diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnections.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnections.java index 319d1edd04b..aab45f4854f 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnections.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnections.java @@ -13,6 +13,7 @@ package org.eclipse.ditto.connectivity.model.signals.commands.query; import java.util.Objects; +import java.util.Optional; import java.util.function.Predicate; import javax.annotation.Nullable; @@ -23,10 +24,12 @@ import org.eclipse.ditto.base.model.json.JsonParsableCommand; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.commands.AbstractCommand; +import org.eclipse.ditto.base.model.signals.commands.WithSelectedFields; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonField; import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.json.JsonPointer; @@ -40,7 +43,7 @@ @Immutable @JsonParsableCommand(typePrefix = ConnectivityCommand.TYPE_PREFIX, name = RetrieveConnections.NAME) public final class RetrieveConnections extends AbstractCommand - implements ConnectivityQueryCommand { + implements ConnectivityQueryCommand, WithSelectedFields { /** * Name of the "Retrieve Connections" command. @@ -54,11 +57,19 @@ public final class RetrieveConnections extends AbstractCommand JSON_IDS_ONLY = JsonFactory.newBooleanFieldDefinition("idsOnly", FieldType.REGULAR, JsonSchemaVersion.V_2); + static final JsonFieldDefinition JSON_SELECTED_FIELDS = + JsonFactory.newStringFieldDefinition("selectedFields", FieldType.REGULAR, + JsonSchemaVersion.V_2); + private final boolean idsOnly; + @Nullable private final JsonFieldSelector selectedFields; - private RetrieveConnections(final boolean idsOnly, final DittoHeaders dittoHeaders) { + private RetrieveConnections(final boolean idsOnly, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { super(TYPE, dittoHeaders); this.idsOnly = idsOnly; + this.selectedFields = selectedFields; } /** @@ -70,7 +81,22 @@ private RetrieveConnections(final boolean idsOnly, final DittoHeaders dittoHeade */ public static RetrieveConnections newInstance(final boolean idsOnly, final DittoHeaders dittoHeaders) { - return new RetrieveConnections(idsOnly, dittoHeaders); + return new RetrieveConnections(idsOnly, null, dittoHeaders); + } + + /** + * Returns a new instance of the retrieve connections command. + * + * @param dittoHeaders provide additional information regarding connections retrieval like a correlation ID. + * @param selectedFields the fields of the JSON representation of the Connection to retrieve. + * @return the instance. + * @throws NullPointerException if any argument is {@code null}. + */ + public static RetrieveConnections newInstance(final boolean idsOnly, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { + + return new RetrieveConnections(idsOnly, selectedFields, dittoHeaders); } /** @@ -100,8 +126,13 @@ public static RetrieveConnections fromJson(final String jsonString, final DittoH */ public static RetrieveConnections fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { final boolean idsOnly = jsonObject.getValueOrThrow(JSON_IDS_ONLY); + final JsonFieldSelector extractedFieldSelector = jsonObject.getValue(JSON_SELECTED_FIELDS) + .map(str -> JsonFactory.newFieldSelector(str, JsonFactory.newParseOptionsBuilder() + .withoutUrlDecoding() + .build())) + .orElse(null); - return new RetrieveConnections(idsOnly, dittoHeaders); + return new RetrieveConnections(idsOnly, extractedFieldSelector, dittoHeaders); } /** @@ -113,6 +144,11 @@ public boolean getIdsOnly() { return idsOnly; } + @Override + public Optional getSelectedFields() { + return Optional.ofNullable(selectedFields); + } + @Override public Category getCategory() { return Category.QUERY; @@ -120,7 +156,7 @@ public Category getCategory() { @Override public RetrieveConnections setDittoHeaders(final DittoHeaders dittoHeaders) { - return new RetrieveConnections(idsOnly, dittoHeaders); + return new RetrieveConnections(idsOnly, selectedFields, dittoHeaders); } @Override @@ -134,6 +170,9 @@ protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final Js final Predicate predicate = jsonSchemaVersion.and(thePredicate); jsonObjectBuilder.set(JSON_IDS_ONLY, idsOnly, predicate); + if (null != selectedFields) { + jsonObjectBuilder.set(JSON_SELECTED_FIELDS, selectedFields.toString(), predicate); + } } @Override @@ -148,12 +187,13 @@ public boolean equals(@Nullable final Object o) { return false; } final RetrieveConnections that = (RetrieveConnections) o; - return Objects.equals(idsOnly, that.idsOnly); + return Objects.equals(idsOnly, that.idsOnly) && + Objects.equals(selectedFields, that.selectedFields); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), idsOnly); + return Objects.hash(super.hashCode(), idsOnly, selectedFields); } @Override @@ -161,6 +201,7 @@ public String toString() { return getClass().getSimpleName() + " [" + super.toString() + ", idsOnly=" + idsOnly + + ", selectedFields=" + selectedFields + "]"; } diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnection.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnection.java index f4ba9a53678..54441b71ece 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnection.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnection.java @@ -15,22 +15,27 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.util.Objects; +import java.util.Optional; import java.util.function.Predicate; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonParsableCommand; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.AbstractCommand; import org.eclipse.ditto.base.model.signals.commands.CommandJsonDeserializer; +import org.eclipse.ditto.base.model.signals.commands.WithSelectedFields; import org.eclipse.ditto.connectivity.model.ConnectionId; import org.eclipse.ditto.connectivity.model.WithConnectionId; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonObjectBuilder; @@ -43,7 +48,8 @@ @Immutable @JsonParsableCommand(typePrefix = ConnectivityCommand.TYPE_PREFIX, name = RetrieveResolvedHonoConnection.NAME) public final class RetrieveResolvedHonoConnection extends AbstractCommand - implements ConnectivityQueryCommand, WithConnectionId, SignalWithEntityId { + implements ConnectivityQueryCommand, WithConnectionId, WithSelectedFields, + SignalWithEntityId { /** * Name of this command. @@ -55,11 +61,19 @@ public final class RetrieveResolvedHonoConnection extends AbstractCommand JSON_SELECTED_FIELDS = + JsonFactory.newStringFieldDefinition("selectedFields", FieldType.REGULAR, + JsonSchemaVersion.V_2); + private final ConnectionId connectionId; + @Nullable private final JsonFieldSelector selectedFields; - private RetrieveResolvedHonoConnection(final ConnectionId connectionId, final DittoHeaders dittoHeaders) { + private RetrieveResolvedHonoConnection(final ConnectionId connectionId, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { super(TYPE, dittoHeaders); this.connectionId = connectionId; + this.selectedFields = selectedFields; } /** @@ -72,7 +86,23 @@ private RetrieveResolvedHonoConnection(final ConnectionId connectionId, final Di */ public static RetrieveResolvedHonoConnection of(final ConnectionId connectionId, final DittoHeaders dittoHeaders) { checkNotNull(connectionId, "connectionId"); - return new RetrieveResolvedHonoConnection(connectionId, dittoHeaders); + return new RetrieveResolvedHonoConnection(connectionId, null, dittoHeaders); + } + + /** + * Returns a new instance of {@code RetrieveResolvedHonoConnection}. + * + * @param connectionId the identifier of the connection to be retrieved. + * @param selectedFields the fields of the JSON representation of the HonoConnection to retrieve. + * @param dittoHeaders the headers of the request. + * @return a new RetrieveResolvedHonoConnection command. + * @throws NullPointerException if any argument is {@code null}. + */ + public static RetrieveResolvedHonoConnection of(final ConnectionId connectionId, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { + checkNotNull(connectionId, "Connection ID"); + return new RetrieveResolvedHonoConnection(connectionId, selectedFields, dittoHeaders); } /** @@ -104,8 +134,12 @@ public static RetrieveResolvedHonoConnection fromJson(final JsonObject jsonObjec return new CommandJsonDeserializer(TYPE, jsonObject).deserialize(() -> { final String readConnectionId = jsonObject.getValueOrThrow(ConnectivityCommand.JsonFields.JSON_CONNECTION_ID); final ConnectionId connectionId = ConnectionId.of(readConnectionId); + final Optional selectedFields = jsonObject.getValue(JSON_SELECTED_FIELDS) + .map(str -> JsonFactory.newFieldSelector(str, JsonFactory.newParseOptionsBuilder() + .withoutUrlDecoding() + .build())); - return of(connectionId, dittoHeaders); + return of(connectionId, selectedFields.orElse(null), dittoHeaders); }); } @@ -116,6 +150,9 @@ protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final Js final Predicate predicate = schemaVersion.and(thePredicate); jsonObjectBuilder.set(ConnectivityCommand.JsonFields.JSON_CONNECTION_ID, String.valueOf(connectionId), predicate); + if (null != selectedFields) { + jsonObjectBuilder.set(JSON_SELECTED_FIELDS, selectedFields.toString(), predicate); + } } @Override @@ -128,9 +165,14 @@ public Category getCategory() { return Category.QUERY; } + @Override + public Optional getSelectedFields() { + return Optional.ofNullable(selectedFields); + } + @Override public RetrieveResolvedHonoConnection setDittoHeaders(final DittoHeaders dittoHeaders) { - return of(connectionId, dittoHeaders); + return of(connectionId, selectedFields, dittoHeaders); } @Override @@ -150,12 +192,13 @@ public boolean equals(@Nullable final Object o) { return false; } final RetrieveResolvedHonoConnection that = (RetrieveResolvedHonoConnection) o; - return Objects.equals(connectionId, that.connectionId); + return Objects.equals(connectionId, that.connectionId) && + Objects.equals(selectedFields, that.selectedFields); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), connectionId); + return Objects.hash(super.hashCode(), connectionId, selectedFields); } @Override @@ -163,6 +206,7 @@ public String toString() { return getClass().getSimpleName() + " [" + super.toString() + ", connectionId=" + connectionId + + ", selectedFields=" + selectedFields + "]"; } diff --git a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionTest.java b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionTest.java index 1137132cc16..eca9da01802 100644 --- a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionTest.java +++ b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionTest.java @@ -24,6 +24,7 @@ import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.connectivity.model.signals.commands.TestConstants; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.json.assertions.DittoJsonAssertions; @@ -52,7 +53,7 @@ public void testHashCodeAndEquals() { public void assertImmutability() { assertInstancesOf(RetrieveConnection.class, areImmutable(), - provided(ConnectionId.class).isAlsoImmutable()); + provided(ConnectionId.class, JsonFieldSelector.class).isAlsoImmutable()); } @Test diff --git a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionsTest.java b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionsTest.java index 375d243bf80..8d70bccc83f 100644 --- a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionsTest.java +++ b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionsTest.java @@ -15,6 +15,7 @@ import static org.eclipse.ditto.json.assertions.DittoJsonAssertions.assertThat; import static org.mutabilitydetector.unittesting.AllowedReason.assumingFields; +import static org.mutabilitydetector.unittesting.AllowedReason.provided; import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; @@ -28,6 +29,7 @@ import org.eclipse.ditto.base.model.signals.commands.GlobalCommandRegistry; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.json.assertions.DittoJsonAssertions; @@ -72,7 +74,8 @@ public void setUp() { public void assertImmutability() { assertInstancesOf(RetrieveConnections.class, areImmutable(), - assumingFields("connectionIds").areSafelyCopiedUnmodifiableCollectionsWithImmutableElements()); + assumingFields("connectionIds").areSafelyCopiedUnmodifiableCollectionsWithImmutableElements(), + provided(JsonFieldSelector.class).isAlsoImmutable()); } @Test diff --git a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnectionTest.java b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnectionTest.java index 94bd12fb22a..5018d59d8af 100644 --- a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnectionTest.java +++ b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnectionTest.java @@ -27,6 +27,7 @@ import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.connectivity.model.signals.commands.TestConstants; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.json.assertions.DittoJsonAssertions; @@ -55,7 +56,7 @@ public void testHashCodeAndEquals() { public void assertImmutability() { assertInstancesOf(RetrieveResolvedHonoConnection.class, areImmutable(), - provided(ConnectionId.class).isAlsoImmutable()); + provided(ConnectionId.class, JsonFieldSelector.class).isAlsoImmutable()); } @Test diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/ConnectivityRootActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/ConnectivityRootActor.java index 899a587fd66..9a3357892a9 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/ConnectivityRootActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/ConnectivityRootActor.java @@ -83,8 +83,6 @@ private ConnectivityRootActor(final ConnectivityConfig connectivityConfig, final var dittoExtensionsConfig = ScopedConfig.dittoExtension(actorSystem.settings().config()); final var enforcerActorPropsFactory = ConnectionEnforcerActorPropsFactory.get(actorSystem, dittoExtensionsConfig); - final var connectionSupervisorProps = - ConnectionSupervisorActor.props(commandForwarder, pubSubMediator, enforcerActorPropsFactory); // Create persistence streaming actor (with no cache) and make it known to pubSubMediator. final ActorRef persistenceStreamingActor = startChildActor(ConnectionPersistenceStreamingActorCreator.ACTOR_NAME, @@ -100,6 +98,10 @@ private ConnectivityRootActor(final ConnectivityConfig connectivityConfig, DittoProtocolSub.get(actorSystem); final MongoReadJournal mongoReadJournal = MongoReadJournal.newInstance(actorSystem); + + final var connectionSupervisorProps = + ConnectionSupervisorActor.props(commandForwarder, pubSubMediator, enforcerActorPropsFactory, + mongoReadJournal); startClusterSingletonActor( PersistencePingActor.props( startConnectionShardRegion(actorSystem, connectionSupervisorProps, clusterConfig), diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/ConnectionConfig.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/ConnectionConfig.java index 04c61b58f74..b23934471b1 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/ConnectionConfig.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/ConnectionConfig.java @@ -20,15 +20,18 @@ import org.eclipse.ditto.base.service.config.supervision.WithSupervisorConfig; import org.eclipse.ditto.edge.service.acknowledgements.AcknowledgementConfig; import org.eclipse.ditto.internal.utils.config.KnownConfigValue; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithActivityCheckConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithSnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.WithCleanupConfig; /** * Provides configuration settings for Connectivity service's connection behaviour. */ @Immutable -public interface ConnectionConfig extends WithSupervisorConfig, WithActivityCheckConfig, WithCleanupConfig { +public interface ConnectionConfig extends WithSupervisorConfig, WithActivityCheckConfig, WithCleanupConfig, + WithSnapshotConfig { /** * Returns the amount of time for how long the connection actor waits for response from client actors. @@ -75,6 +78,13 @@ public interface ConnectionConfig extends WithSupervisorConfig, WithActivityChec */ SnapshotConfig getSnapshotConfig(); + /** + * Returns the config of the connection event journal behaviour. + * + * @return the config. + */ + EventConfig getEventConfig(); + /** * Returns the maximum number of Targets within a connection. * diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/DefaultConnectionConfig.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/DefaultConnectionConfig.java index b30128e9885..f06c6971097 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/DefaultConnectionConfig.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/DefaultConnectionConfig.java @@ -25,7 +25,9 @@ import org.eclipse.ditto.internal.utils.config.ConfigWithFallback; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultActivityCheckConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultEventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultSnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.CleanupConfig; @@ -47,6 +49,7 @@ public final class DefaultConnectionConfig implements ConnectionConfig { private final String blockedHostRegex; private final SupervisorConfig supervisorConfig; private final SnapshotConfig snapshotConfig; + private final EventConfig eventConfig; private final DefaultAcknowledgementConfig acknowledgementConfig; private final CleanupConfig cleanupConfig; private final Amqp10Config amqp10Config; @@ -74,6 +77,7 @@ private DefaultConnectionConfig(final ConfigWithFallback config) { blockedHostRegex = config.getString(ConnectionConfigValue.BLOCKED_HOST_REGEX.getConfigPath()); supervisorConfig = DefaultSupervisorConfig.of(config); snapshotConfig = DefaultSnapshotConfig.of(config); + eventConfig = DefaultEventConfig.of(config); acknowledgementConfig = DefaultAcknowledgementConfig.of(config); cleanupConfig = CleanupConfig.of(config); amqp10Config = DefaultAmqp10Config.of(config); @@ -153,6 +157,11 @@ public SnapshotConfig getSnapshotConfig() { return snapshotConfig; } + @Override + public EventConfig getEventConfig() { + return eventConfig; + } + @Override public Integer getMaxNumberOfTargets() { return maxNumberOfTargets; @@ -246,6 +255,7 @@ public boolean equals(final Object o) { Objects.equals(blockedHostRegex, that.blockedHostRegex) && Objects.equals(supervisorConfig, that.supervisorConfig) && Objects.equals(snapshotConfig, that.snapshotConfig) && + Objects.equals(eventConfig, that.eventConfig) && Objects.equals(acknowledgementConfig, that.acknowledgementConfig) && Objects.equals(cleanupConfig, that.cleanupConfig) && Objects.equals(amqp10Config, that.amqp10Config) && @@ -266,7 +276,7 @@ public boolean equals(final Object o) { @Override public int hashCode() { return Objects.hash(clientActorAskTimeout, clientActorRestartsBeforeEscalation, allowedHostnames, - blockedHostnames, blockedSubnets, blockedHostRegex, supervisorConfig, snapshotConfig, + blockedHostnames, blockedSubnets, blockedHostRegex, supervisorConfig, snapshotConfig, eventConfig, acknowledgementConfig, cleanupConfig, maxNumberOfTargets, maxNumberOfSources, activityCheckConfig, fieldsEncryptionConfig, amqp10Config, amqp091Config, mqttConfig, kafkaConfig, httpPushConfig, ackLabelDeclareInterval, priorityUpdateInterval, shutdownTimeout, allClientActorsOnOneNode); @@ -283,6 +293,7 @@ public String toString() { ", blockedHostRegex=" + blockedHostRegex + ", supervisorConfig=" + supervisorConfig + ", snapshotConfig=" + snapshotConfig + + ", eventConfig=" + eventConfig + ", acknowledgementConfig=" + acknowledgementConfig + ", cleanUpConfig=" + cleanupConfig + ", amqp10Config=" + amqp10Config + diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ConnectionExistenceChecker.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ConnectionExistenceChecker.java new file mode 100644 index 00000000000..24fa1f91095 --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ConnectionExistenceChecker.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.service.enforcement.pre; + +import java.util.concurrent.CompletionStage; + +import org.eclipse.ditto.connectivity.api.ConnectivityMessagingConstants; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; +import org.eclipse.ditto.internal.utils.cache.entry.Entry; +import org.eclipse.ditto.internal.utils.cluster.ShardRegionProxyActorFactory; +import org.eclipse.ditto.internal.utils.cluster.config.DefaultClusterConfig; +import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; +import org.eclipse.ditto.policies.enforcement.config.DefaultEnforcementConfig; +import org.eclipse.ditto.policies.enforcement.config.EnforcementConfig; + +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; + +/** + * checks for the existence of connections. + */ +final class ConnectionExistenceChecker { + + public static final String ENFORCEMENT_CACHE_DISPATCHER = "enforcement-cache-dispatcher"; + + private final AsyncCacheLoader> connectionIdLoader; + private final ActorSystem actorSystem; + + ConnectionExistenceChecker(final ActorSystem actorSystem) { + this.actorSystem = actorSystem; + final var enforcementConfig = DefaultEnforcementConfig.of( + DefaultScopedConfig.dittoScoped(actorSystem.settings().config())); + connectionIdLoader = getConnectionIdLoader(actorSystem, enforcementConfig); + } + + private AsyncCacheLoader> getConnectionIdLoader( + final ActorSystem actorSystem, + final EnforcementConfig enforcementConfig) { + + final var clusterConfig = DefaultClusterConfig.of(actorSystem.settings().config().getConfig("ditto.cluster")); + final ShardRegionProxyActorFactory shardRegionProxyActorFactory = + ShardRegionProxyActorFactory.newInstance(actorSystem, clusterConfig); + + final ActorRef connectionShardRegion = shardRegionProxyActorFactory.getShardRegionProxyActor( + ConnectivityMessagingConstants.CLUSTER_ROLE, ConnectivityMessagingConstants.SHARD_REGION); + return new PreEnforcementConnectionIdCacheLoader(enforcementConfig.getAskWithRetryConfig(), + actorSystem.getScheduler(), + connectionShardRegion); + } + + public CompletionStage checkExistence(final ModifyConnection signal) { + try { + return connectionIdLoader.asyncLoad(signal.getEntityId(), + actorSystem.dispatchers().lookup(ENFORCEMENT_CACHE_DISPATCHER)) + .thenApply(Entry::exists); + } catch (final Exception e) { + throw new IllegalStateException("Could not load connection via connectionIdLoader", e); + } + } + +} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ModifyToCreateConnectionTransformer.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ModifyToCreateConnectionTransformer.java new file mode 100644 index 00000000000..5d713fe5781 --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ModifyToCreateConnectionTransformer.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.service.enforcement.pre; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.base.service.signaltransformer.SignalTransformer; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnection; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; + +import com.typesafe.config.Config; + +import akka.actor.ActorSystem; + +/** + * Transforms a ModifyConnection into a CreateConnection if the connection does not exist already. + */ +public final class ModifyToCreateConnectionTransformer implements SignalTransformer { + + private final ConnectionExistenceChecker existenceChecker; + + @SuppressWarnings("unused") + ModifyToCreateConnectionTransformer(final ActorSystem actorSystem, final Config config) { + this(new ConnectionExistenceChecker(actorSystem)); + } + + ModifyToCreateConnectionTransformer(final ConnectionExistenceChecker existenceChecker) { + this.existenceChecker = existenceChecker; + } + + @Override + public CompletionStage> apply(final Signal signal) { + if (signal instanceof ModifyConnection modifyConnection) { + return existenceChecker.checkExistence(modifyConnection) + .thenApply(exists -> { + if (Boolean.FALSE.equals(exists)) { + return CreateConnection.of( + modifyConnection.getConnection(), + modifyConnection.getDittoHeaders() + ); + } else { + return modifyConnection; + } + }); + } else { + return CompletableFuture.completedStage(signal); + } + } + +} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/PreEnforcementConnectionIdCacheLoader.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/PreEnforcementConnectionIdCacheLoader.java new file mode 100644 index 00000000000..8968c77b109 --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/PreEnforcementConnectionIdCacheLoader.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.service.enforcement.pre; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; + +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; +import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection; +import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionResponse; +import org.eclipse.ditto.internal.utils.cache.entry.Entry; +import org.eclipse.ditto.internal.utils.cacheloaders.ActorAskCacheLoader; +import org.eclipse.ditto.internal.utils.cacheloaders.config.AskWithRetryConfig; +import org.eclipse.ditto.json.JsonFieldSelector; + +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; + +import akka.actor.ActorRef; +import akka.actor.Scheduler; + +/** + * Cache loader used for Connection existence check in pre-enforcement. + */ +final class PreEnforcementConnectionIdCacheLoader implements + AsyncCacheLoader> { + + private final ActorAskCacheLoader, ConnectionId> delegate; + + /** + * Constructor. + * + * @param askWithRetryConfig the configuration for the "ask with retry" pattern applied for the cache loader. + * @param scheduler the scheduler to use for the "ask with retry" for retries. + * @param shardRegionProxy the shard-region-proxy. + */ + public PreEnforcementConnectionIdCacheLoader(final AskWithRetryConfig askWithRetryConfig, + final Scheduler scheduler, + final ActorRef shardRegionProxy) { + + delegate = ActorAskCacheLoader.forShard(askWithRetryConfig, + scheduler, + ConnectivityConstants.ENTITY_TYPE, + shardRegionProxy, + connectionId -> RetrieveConnection.of(connectionId, JsonFieldSelector.newInstance("id"), DittoHeaders.empty()), + PreEnforcementConnectionIdCacheLoader::handleRetrieveConnectionResponse); + } + + @Override + public CompletableFuture> asyncLoad(final ConnectionId key, final Executor executor) { + return delegate.asyncLoad(key, executor); + } + + private static Entry handleRetrieveConnectionResponse(final Object response) { + + if (response instanceof RetrieveConnectionResponse retrieveConnectionResponse) { + final ConnectionId connectionId = retrieveConnectionResponse.getEntityId(); + return Entry.of(-1, connectionId); + } else if (response instanceof ConnectionNotAccessibleException) { + return Entry.nonexistent(); + } else { + throw new IllegalStateException("expect RetrieveConnectionResponse, got: " + response); + } + } + +} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/package-info.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/package-info.java new file mode 100644 index 00000000000..35e7d14ecc6 --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/package-info.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.eclipse.ditto.utils.jsr305.annotations.AllParametersAndReturnValuesAreNonnullByDefault +package org.eclipse.ditto.connectivity.service.enforcement.pre; diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/CloudEventsMapper.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/CloudEventsMapper.java index b863c083fe7..1c5d913dddb 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/CloudEventsMapper.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/CloudEventsMapper.java @@ -58,7 +58,7 @@ public final class CloudEventsMapper extends AbstractMessageMapper { private static final String TYPE = "type"; private static final String OUTBOUNDTYPE = "org.eclipse.ditto.outbound"; private static final String OUTBOUNDSPECVERSION = "1.0"; - private static final String OUTBOUNDSOURCE = "https://github.com/eclipse/ditto"; + private static final String OUTBOUNDSOURCE = "https://github.com/eclipse-ditto/ditto"; private static final String DATA = "data"; private static final String DATA_BASE64 = "data_base64"; private static final String OUTBOUND_DATA_CONTENT_TYPE = "datacontenttype"; diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseClientActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseClientActor.java index dddec5cbe04..0c026c0ced7 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseClientActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseClientActor.java @@ -61,7 +61,9 @@ import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; import org.eclipse.ditto.base.model.signals.WithType; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; import org.eclipse.ditto.connectivity.api.BaseClientState; import org.eclipse.ditto.connectivity.api.InboundSignal; import org.eclipse.ditto.connectivity.api.OutboundSignal; @@ -114,6 +116,7 @@ import org.eclipse.ditto.connectivity.service.util.ConnectionPubSub; import org.eclipse.ditto.connectivity.service.util.ConnectivityMdcEntryKey; import org.eclipse.ditto.edge.service.headers.DittoHeadersValidator; +import org.eclipse.ditto.edge.service.streaming.StreamingSubscriptionManager; import org.eclipse.ditto.internal.models.signal.correlation.MatchingValidationResult; import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory; import org.eclipse.ditto.internal.utils.akka.logging.ThreadSafeDittoLoggingAdapter; @@ -178,8 +181,10 @@ public abstract class BaseClientActor extends AbstractFSMWithStash inboundMappingSink; private ActorRef outboundDispatchingActor; private ActorRef subscriptionManager; + private ActorRef streamingSubscriptionManager; private ActorRef tunnelActor; private int ackregatorCount = 0; private boolean shuttingDown = false; @@ -325,6 +331,7 @@ protected void init() { inboundMappingSink = getInboundMappingSink(protocolAdapter, inboundDispatchingSink); subscriptionManager = startSubscriptionManager(commandForwarderActorSelection, connectivityConfig().getClientConfig()); + streamingSubscriptionManager = startStreamingSubscriptionManager(commandForwarderActorSelection, connectivityConfig().getClientConfig()); if (connection.getSshTunnel().map(SshTunnel::isEnabled).orElse(false)) { tunnelActor = childActorNanny.startChildActor(SshTunnelActor.ACTOR_NAME, @@ -505,6 +512,7 @@ protected void cleanupFurtherResourcesOnConnectionTimeout(final BaseClientState */ protected FSMStateFunctionBuilder inAnyState() { return matchEvent(RetrieveConnectionMetrics.class, (command, data) -> retrieveConnectionMetrics(command)) + .event(StreamingSubscriptionCommand.class, this::forwardStreamingSubscriptionCommand) .event(ThingSearchCommand.class, this::forwardThingSearchCommand) .event(RetrieveConnectionStatus.class, this::retrieveConnectionStatus) .event(ResetConnectionMetrics.class, this::resetConnectionMetrics) @@ -1435,8 +1443,8 @@ private FSM.State retrieveConnectionStatus(fina " Forwarding to consumers and publishers.", command.getEntityId(), sender); - // only one PublisherActor is started for all targets (if targets are present) - final int numberOfProducers = connection.getTargets().isEmpty() ? 0 : 1; + // only one PublisherActor is started for all targets - and it always is started + final int numberOfProducers = 1; final int numberOfConsumers = determineNumberOfConsumers(); int expectedNumberOfChildren = numberOfProducers + numberOfConsumers; if (getSshTunnelState().isEnabled()) { @@ -1716,6 +1724,8 @@ private FSM.State handleInboundSignal(final Inb final Signal signal = inboundSignal.getSignal(); if (signal instanceof WithSubscriptionId) { dispatchSearchCommand((WithSubscriptionId) signal); + } else if (signal instanceof WithStreamingSubscriptionId) { + dispatchStreamingSubscriptionCommand((WithStreamingSubscriptionId) signal); } else { final var entityId = tryExtractEntityId(signal).orElseThrow(); connectionPubSub.publishSignal(inboundSignal.asDispatched(), connectionId(), entityId, getSender()); @@ -1746,6 +1756,24 @@ private void dispatchSearchCommand(final WithSubscriptionId searchCommand) { } } + private void dispatchStreamingSubscriptionCommand( + final WithStreamingSubscriptionId streamingSubscriptionCommand) { + + final String subscriptionId = streamingSubscriptionCommand.getSubscriptionId(); + if (subscriptionId.length() > subscriptionIdPrefixLength) { + final var prefix = subscriptionId.substring(0, subscriptionIdPrefixLength); + connectionPubSub.publishSignal(streamingSubscriptionCommand, connectionId(), prefix, ActorRef.noSender()); + } else { + // command is invalid or outdated, dropping. + logger.withCorrelationId(streamingSubscriptionCommand) + .info("Dropping streaming subscription command with invalid subscription ID: <{}>", + streamingSubscriptionCommand); + connectionLogger.failure(InfoProviderFactory.forSignal(streamingSubscriptionCommand), + "Dropping streaming subscription command with invalid subscription ID: " + + streamingSubscriptionCommand.getSubscriptionId()); + } + } + private Instant getInConnectionStatusSince() { return stateData().getInConnectionStatusSince(); } @@ -1891,6 +1919,19 @@ private ActorRef startSubscriptionManager(final ActorSelection proxyActor, final return getContext().actorOf(props, SubscriptionManager.ACTOR_NAME); } + /** + * Start the streaming subscription manager. Requires MessageMappingProcessorActor to be started to work. + * Creates an actor materializer. + * + * @return reference of the streaming subscription manager. + */ + private ActorRef startStreamingSubscriptionManager(final ActorSelection proxyActor, final ClientConfig clientConfig) { + final var mat = Materializer.createMaterializer(this::getContext); + final var props = StreamingSubscriptionManager.props(clientConfig.getSubscriptionManagerTimeout(), + proxyActor, mat); + return getContext().actorOf(props, StreamingSubscriptionManager.ACTOR_NAME); + } + private FSM.State forwardThingSearchCommand(final WithDittoHeaders command, final BaseClientData data) { // Tell subscriptionManager to send search events to messageMappingProcessorActor. @@ -1906,6 +1947,19 @@ private FSM.State forwardThingSearchCommand(fin return stay(); } + private FSM.State forwardStreamingSubscriptionCommand( + final StreamingSubscriptionCommand command, + final BaseClientData data) { + // Tell subscriptionManager to send streaming subscription events to messageMappingProcessorActor. + if (stateName() == CONNECTED) { + streamingSubscriptionManager.tell(command, outboundDispatchingActor); + } else { + logger.withCorrelationId(command) + .debug("Client state <{}> is not CONNECTED; dropping <{}>", stateName(), command); + } + return stay(); + } + private FSM.State resubscribe(final Control trigger, final BaseClientData data) { subscribeAndDeclareAcknowledgementLabels(dryRun, true); startSubscriptionRefreshTimer(); diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseConsumerActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseConsumerActor.java index fe46ef752b7..bf45d23cc4c 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseConsumerActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseConsumerActor.java @@ -186,18 +186,8 @@ private void prepareResponseHandler(final AcknowledgeableMessage acknowledgeable } else { // don't count this as "failure" in the "source consumed" metric as the consumption // itself was successful - final var dittoRuntimeException = DittoRuntimeException.asDittoRuntimeException(error, rootCause -> { - - // Redeliver and pray this unexpected error goes away - log().debug("Rejecting [redeliver=true] due to error <{}>. " + - "ResponseCollector=<{}>", rootCause, responseCollector); - ackTimer.tag(getAckSuccessTag(false)); - ackTimer.tag(getAckRedeliverTag(true)); - ackTimer.stop(); - ackCounter.decrement(); - acknowledgeableMessage.reject(true); - return null; - }); + final var dittoRuntimeException = DittoRuntimeException + .asDittoRuntimeException(error, rootCause -> null); if (dittoRuntimeException != null) { if (isConsideredSuccess(dittoRuntimeException)) { ackTimer.tag(getAckSuccessTag(true)); diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BasePublisherActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BasePublisherActor.java index 960583434c8..9583bd4e155 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BasePublisherActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BasePublisherActor.java @@ -48,6 +48,7 @@ import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.model.signals.acks.Acknowledgements; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.connectivity.api.ExternalMessage; import org.eclipse.ditto.connectivity.api.OutboundSignal; import org.eclipse.ditto.connectivity.model.Connection; @@ -356,7 +357,7 @@ private Stream sendMappedOutboundSignal(final OutboundSignal.M private Optional getSendingContext(final OutboundSignal.Mapped mappedOutboundSignal) { final Optional result; - if (isResponseOrErrorOrSearchEvent(mappedOutboundSignal)) { + if (isResponseOrErrorOrStreamingEvent(mappedOutboundSignal)) { final Signal source = mappedOutboundSignal.getSource(); final DittoHeaders dittoHeaders = source.getDittoHeaders(); result = dittoHeaders.getReplyTarget() @@ -369,17 +370,19 @@ private Optional getSendingContext(final OutboundSignal.Mapped m } /** - * Checks whether the passed in {@code outboundSignal} is a response or an error or a search event. + * Checks whether the passed in {@code outboundSignal} is a response or an error or a streaming event + * (including search events). * Those messages are supposed to be published at the reply target of the source whence the original command came. * * @param outboundSignal the OutboundSignal to check. * @return {@code true} if the OutboundSignal is a response or an error, {@code false} otherwise */ - private static boolean isResponseOrErrorOrSearchEvent(final OutboundSignal.Mapped outboundSignal) { + private static boolean isResponseOrErrorOrStreamingEvent(final OutboundSignal.Mapped outboundSignal) { final ExternalMessage externalMessage = outboundSignal.getExternalMessage(); return externalMessage.isResponse() || externalMessage.isError() || - outboundSignal.getSource() instanceof SubscriptionEvent; + outboundSignal.getSource() instanceof SubscriptionEvent || + outboundSignal.getSource() instanceof StreamingSubscriptionEvent; } private Optional getReplyTargetByIndex(final int replyTargetIndex) { diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/ConnectionIdsRetrievalActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/ConnectionIdsRetrievalActor.java index 7e7e133c9a7..9373a78fb35 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/ConnectionIdsRetrievalActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/ConnectionIdsRetrievalActor.java @@ -40,6 +40,7 @@ import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory; import org.eclipse.ditto.internal.utils.cluster.DistPubSubAccess; import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; +import org.eclipse.ditto.internal.utils.persistentactors.EmptyEvent; import akka.NotUsed; import akka.actor.AbstractActor; @@ -114,6 +115,12 @@ private static boolean isDeleted(final Document document) { .orElse(true); } + private static boolean isNotEmptyEvent(final Document document) { + return Optional.ofNullable(document.getString(MongoReadJournal.J_EVENT_MANIFEST)) + .map(manifest -> !EmptyEvent.TYPE.equals(manifest)) + .orElse(false); + } + private static boolean isNotDeleted(final Document document) { return Optional.ofNullable(document.getString(MongoReadJournal.J_EVENT_MANIFEST)) .map(manifest -> !ConnectionDeleted.TYPE.equals(manifest)) @@ -166,6 +173,7 @@ private void getAllConnectionIDs(final WithDittoHeaders cmd) { final Source idsFromSnapshots = getIdsFromSnapshotsSource(); final Source idsFromJournal = persistenceIdsFromJournalSourceSupplier.get() .filter(ConnectionIdsRetrievalActor::isNotDeleted) + .filter(ConnectionIdsRetrievalActor::isNotEmptyEvent) .map(document -> document.getString(MongoReadJournal.J_EVENT_PID)); final CompletionStage retrieveAllConnectionIdsResponse = diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/InboundDispatchingSink.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/InboundDispatchingSink.java index afe0da9db33..53d2ae0f277 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/InboundDispatchingSink.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/InboundDispatchingSink.java @@ -44,11 +44,13 @@ import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.model.signals.acks.Acknowledgements; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; import org.eclipse.ditto.base.model.signals.commands.ErrorResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.connectivity.api.ExternalMessage; import org.eclipse.ditto.connectivity.api.InboundSignal; import org.eclipse.ditto.connectivity.api.MappedInboundExternalMessage; @@ -508,6 +510,8 @@ private PartialFunction, Stream> dispatchResponsesAndS ) .match(CreateSubscription.class, cmd -> forwardToConnectionActor(cmd, sender)) .match(WithSubscriptionId.class, cmd -> forwardToClientActor(cmd, sender)) + .match(SubscribeForPersistedEvents.class, cmd -> forwardToConnectionActor(cmd, sender)) + .match(WithStreamingSubscriptionId.class, cmd -> forwardToClientActor(cmd, sender)) .matchAny(baseSignal -> ackregatorStarter.preprocess(baseSignal, (signal, isAckRequesting) -> Stream.of(new IncomingSignal(signal, getReturnAddress(sender, isAckRequesting, signal), @@ -659,7 +663,7 @@ private Stream forwardToClientActor(final Signal signal, @Nullable fin return Stream.empty(); } - private Stream forwardToConnectionActor(final CreateSubscription command, @Nullable final ActorRef sender) { + private Stream forwardToConnectionActor(final Command command, @Nullable final ActorRef sender) { connectionActor.tell(command, sender); return Stream.empty(); diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundDispatchingActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundDispatchingActor.java index 630147a10f0..3e63e41d785 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundDispatchingActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundDispatchingActor.java @@ -31,6 +31,7 @@ import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.connectivity.api.InboundSignal; import org.eclipse.ditto.connectivity.api.OutboundSignalFactory; import org.eclipse.ditto.connectivity.model.Target; @@ -79,6 +80,7 @@ public Receive createReceive() { .match(InboundSignal.class, this::inboundSignal) .match(CommandResponse.class, this::forwardWithoutCheck) .match(SubscriptionEvent.class, this::forwardWithoutCheck) + .match(StreamingSubscriptionEvent.class, this::forwardWithoutCheck) .match(DittoRuntimeException.class, this::forwardWithoutCheck) .match(Signal.class, this::handleSignal) .matchAny(message -> logger.warning("Unknown message: <{}>", message)) diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundMappingProcessorActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundMappingProcessorActor.java index 96ee126d233..869909f8b5b 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundMappingProcessorActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundMappingProcessorActor.java @@ -349,13 +349,21 @@ protected OutboundSignalWithSender mapMessage(final OutboundSignal message) { } @Override - protected void messageDiscarded(OutboundSignal message, QueueOfferResult result) { + protected void messageDiscarded(final OutboundSignal message, final QueueOfferResult result) { + final Set monitorsForOutboundSignal = + getMonitorsForOutboundSignal(message, MAPPED, LogType.MAPPED, responseMappedMonitor); if (QueueOfferResult.dropped().equals(result)) { - responseDispatchedMonitor.failure(message.getSource(), "Message is dropped as a result of backpressure strategy!"); + monitorsForOutboundSignal.forEach(monitor -> + monitor.failure(message.getSource(), "Message is dropped as a result of backpressure strategy!") + ); } else if (result instanceof final QueueOfferResult.Failure failure) { - responseDispatchedMonitor.failure(message.getSource(), "Enqueue failed! - failure: {}", failure.cause()); + monitorsForOutboundSignal.forEach(monitor -> + monitor.failure(message.getSource(), "Enqueue failed! - failure: {}", failure.cause()) + ); } else { - responseDispatchedMonitor.failure(message.getSource(), "Enqueue failed without acknowledgement!"); + monitorsForOutboundSignal.forEach(monitor -> + monitor.failure(message.getSource(), "Enqueue failed without acknowledgement!") + ); } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactory.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactory.java index 18c806b30fc..99043999f82 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactory.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactory.java @@ -16,6 +16,8 @@ import java.text.MessageFormat; import java.util.Set; +import org.eclipse.ditto.connectivity.model.Connection; +import org.eclipse.ditto.connectivity.model.ConnectionId; import org.eclipse.ditto.connectivity.model.HonoAddressAlias; import org.eclipse.ditto.connectivity.model.UserPasswordCredentials; import org.eclipse.ditto.connectivity.service.config.DefaultHonoConfig; @@ -33,6 +35,8 @@ public final class DefaultHonoConnectionFactory extends HonoConnectionFactory { private final HonoConfig honoConfig; + private ConnectionId connectionId; + /** * Constructs a {@code DefaultHonoConnectionFactory} for the specified arguments. * @@ -44,6 +48,11 @@ public DefaultHonoConnectionFactory(final ActorSystem actorSystem, final Config honoConfig = new DefaultHonoConfig(actorSystem); } + @Override + protected void preConversion(final Connection honoConnection) { + connectionId = honoConnection.getId(); + } + @Override public URI getBaseUri() { return honoConfig.getBaseUri(); @@ -76,12 +85,14 @@ protected UserPasswordCredentials getCredentials() { @Override protected String resolveSourceAddress(final HonoAddressAlias honoAddressAlias) { - return MessageFormat.format("hono.{0}", honoAddressAlias.getAliasValue()); + return MessageFormat.format("hono.{0}.{1}", + honoAddressAlias.getAliasValue(), connectionId); } @Override protected String resolveTargetAddress(final HonoAddressAlias honoAddressAlias) { - return MessageFormat.format("hono.{0}/'{{thing:id}}'", honoAddressAlias.getAliasValue()); + return MessageFormat.format("hono.{0}.{1}/'{{thing:id}}'", + honoAddressAlias.getAliasValue(), connectionId); } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/HonoConnectionFactory.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/HonoConnectionFactory.java index 3057a0ea4d9..8552aee4b5a 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/HonoConnectionFactory.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/HonoConnectionFactory.java @@ -16,6 +16,8 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.util.Collection; import java.util.LinkedHashMap; @@ -150,7 +152,8 @@ protected void preConversion(final Connection honoConnection) { private String combineUriWithCredentials(final String uri, final UserPasswordCredentials credentials) { return uri.replaceFirst("(\\S+://)(\\S+)", - "$1" + credentials.getUsername() + ":" + credentials.getPassword() + "@$2"); + "$1" + URLEncoder.encode(credentials.getUsername(), StandardCharsets.UTF_8) + ":" + + URLEncoder.encode(credentials.getPassword(), StandardCharsets.UTF_8) + "@$2"); } private Map makeupSpecificConfig(final Connection connection) { diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/client/BaseGenericMqttSubscribingClient.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/client/BaseGenericMqttSubscribingClient.java index 7d1f35158b6..4ec4fc9ad48 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/client/BaseGenericMqttSubscribingClient.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/client/BaseGenericMqttSubscribingClient.java @@ -226,9 +226,13 @@ protected Single sendSubscribe(final Mqtt3RxClient mqtt3RxCli @Override Completable sendUnsubscribe(final Mqtt3RxClient mqtt3RxClient, final MqttTopicFilter... mqttTopicFilters) { - final var unsubscribe = - Mqtt3Unsubscribe.builder().addTopicFilters(mqttTopicFilters).build(); - return mqtt3RxClient.unsubscribe(unsubscribe); + if (mqttTopicFilters.length == 0) { + return Completable.complete(); + } else { + final var unsubscribe = + Mqtt3Unsubscribe.builder().addTopicFilters(mqttTopicFilters).build(); + return mqtt3RxClient.unsubscribe(unsubscribe); + } } @Override diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformer.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformer.java index ebc5ee326ea..65f84b46b3b 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformer.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformer.java @@ -236,6 +236,7 @@ private static Set getUserPropertiesOrEmptySet( ) { return externalMessageHeaders.stream() .filter(header -> !KNOWN_MQTT_HEADER_NAMES.contains(header.getKey())) + .filter(header -> header.getValue() != null && !header.getValue().isBlank()) .map(header -> { final var headerKey = header.getKey(); final String headerValue; diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActor.java index ebed209c9ef..a55e3fb1979 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActor.java @@ -45,6 +45,7 @@ import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.connectivity.api.BaseClientState; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionId; @@ -55,6 +56,7 @@ import org.eclipse.ditto.connectivity.model.ConnectivityStatus; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommandInterceptor; import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionFailedException; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionHistoryNotAccessibleException; import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CheckConnectionLogsActive; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CloseConnection; @@ -198,12 +200,13 @@ public final class ConnectionPersistenceActor @Nullable private Instant recoveredAt; ConnectionPersistenceActor(final ConnectionId connectionId, + final MongoReadJournal mongoReadJournal, final ActorRef commandForwarderActor, final ActorRef pubSubMediator, final Trilean allClientActorsOnOneNode, final Config connectivityConfigOverwrites) { - super(connectionId); + super(connectionId, mongoReadJournal); this.actorSystem = context().system(); cluster = Cluster.get(actorSystem); final Config dittoExtensionConfig = ScopedConfig.dittoExtension(actorSystem.settings().config()); @@ -263,18 +266,21 @@ protected DittoDiagnosticLoggingAdapter createLogger() { * Creates Akka configuration object for this actor. * * @param connectionId the connection ID. - * @param commandForwarderActor the actor used to send signals into the ditto cluster.. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the connection. + * @param commandForwarderActor the actor used to send signals into the ditto cluster. + * @param pubSubMediator pub-sub-mediator for the shutdown behavior. * @param pubSubMediator the pubSubMediator * @param connectivityConfigOverwrites the overwrites for the connectivity config for the given connection. * @return the Akka configuration Props object. */ public static Props props(final ConnectionId connectionId, + final MongoReadJournal mongoReadJournal, final ActorRef commandForwarderActor, final ActorRef pubSubMediator, final Config connectivityConfigOverwrites ) { - return Props.create(ConnectionPersistenceActor.class, connectionId, - commandForwarderActor, pubSubMediator, Trilean.UNKNOWN, connectivityConfigOverwrites); + return Props.create(ConnectionPersistenceActor.class, connectionId, mongoReadJournal, + commandForwarderActor, pubSubMediator,Trilean.UNKNOWN, connectivityConfigOverwrites); } /** @@ -350,6 +356,16 @@ protected DittoRuntimeExceptionBuilder newNotAccessibleExceptionBuilder() { return ConnectionNotAccessibleException.newBuilder(entityId); } + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final long revision) { + return ConnectionHistoryNotAccessibleException.newBuilder(entityId, revision); + } + + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final Instant timestamp) { + return ConnectionHistoryNotAccessibleException.newBuilder(entityId, timestamp); + } + @Override protected void publishEvent(@Nullable final Connection previousEntity, final ConnectivityEvent event) { if (event instanceof ConnectionDeleted) { @@ -564,6 +580,17 @@ public void onMutation(final Command command, final ConnectivityEvent even } } + @Override + public void onStagedMutation(final Command command, final CompletionStage> event, + final CompletionStage response, final boolean becomeCreated, + final boolean becomeDeleted) { + if (command instanceof StagedCommand stagedCommand) { + interpretStagedCommand(stagedCommand.withSenderUnlessDefined(getSender())); + } else { + super.onStagedMutation(command, event, response, becomeCreated, becomeDeleted); + } + } + @Override protected void checkForActivity(final CheckForActivity trigger) { if (isDesiredStateOpen()) { @@ -665,6 +692,9 @@ protected Receive matchAnyAfterInitialization() { // CreateSubscription is a ThingSearchCommand, but it is created in InboundDispatchingSink from an // adaptable and directly sent to this actor: .match(CreateSubscription.class, this::startThingSearchSession) + // SubscribeForPersistedEvents is created in InboundDispatchingSink from an + // adaptable and directly sent to this actor: + .match(SubscribeForPersistedEvents.class, this::startStreamingSubscriptionSession) .matchEquals(Control.CHECK_LOGGING_ACTIVE, this::checkLoggingEnabled) .matchEquals(Control.TRIGGER_UPDATE_PRIORITY, this::triggerUpdatePriority) .match(UpdatePriority.class, this::updatePriority) @@ -725,6 +755,17 @@ private void startThingSearchSession(final CreateSubscription command) { augmentWithPrefixAndForward(command, entity.getClientCount()); } + private void startStreamingSubscriptionSession(final SubscribeForPersistedEvents command) { + if (entity == null) { + logDroppedSignal(command, command.getType(), "No Connection configuration available."); + return; + } + log.debug("Forwarding <{}> to client actors.", command); + // compute the next prefix according to subscriptionCounter and the currently configured client actor count + // ignore any "prefix" field from the command + augmentWithPrefixAndForward(command, entity.getClientCount()); + } + private void augmentWithPrefixAndForward(final CreateSubscription createSubscription, final int clientCount) { subscriptionCounter = (subscriptionCounter + 1) % Math.max(1, clientCount); final var prefix = getPrefix(getSubscriptionPrefixLength(clientCount), subscriptionCounter); @@ -732,6 +773,14 @@ private void augmentWithPrefixAndForward(final CreateSubscription createSubscrip connectionPubSub.publishSignal(commandWithPrefix, entityId, prefix, ActorRef.noSender()); } + private void augmentWithPrefixAndForward(final SubscribeForPersistedEvents subscribeForPersistedEvents, + final int clientCount) { + subscriptionCounter = (subscriptionCounter + 1) % Math.max(1, clientCount); + final var prefix = getPrefix(getSubscriptionPrefixLength(clientCount), subscriptionCounter); + final var commandWithPrefix = subscribeForPersistedEvents.setPrefix(prefix); + connectionPubSub.publishSignal(commandWithPrefix, entityId, prefix, ActorRef.noSender()); + } + private static String getPrefix(final int prefixLength, final int subscriptionCounter) { final var prefixPattern = MessageFormat.format("%0{0,number}X", prefixLength); return String.format(prefixPattern, subscriptionCounter); diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionSupervisorActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionSupervisorActor.java index 05d69b8560b..fc62e17cdc2 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionSupervisorActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionSupervisorActor.java @@ -31,6 +31,7 @@ import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.service.actors.ShutdownBehaviour; import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOffConfig; +import org.eclipse.ditto.base.service.config.supervision.LocalAskTimeoutConfig; import org.eclipse.ditto.connectivity.model.ConnectionId; import org.eclipse.ditto.connectivity.model.ConnectionType; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; @@ -41,6 +42,7 @@ import org.eclipse.ditto.connectivity.service.config.DittoConnectivityConfig; import org.eclipse.ditto.connectivity.service.enforcement.ConnectionEnforcerActorPropsFactory; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import com.typesafe.config.Config; @@ -71,12 +73,6 @@ public final class ConnectionSupervisorActor private static final Duration MAX_CONFIG_RETRIEVAL_DURATION = Duration.ofSeconds(5); - /** - * For connectivity, this local ask timeout has to be higher as e.g. "openConnection" commands performed in a - * "staged" way will lead to quite some response times. - */ - private static final Duration CONNECTIVITY_DEFAULT_LOCAL_ASK_TIMEOUT = Duration.ofSeconds(50); - private static final SupervisorStrategy SUPERVISOR_STRATEGY = new OneForOneStrategy(true, DeciderBuilder.match(JMSRuntimeException.class, e -> @@ -95,10 +91,12 @@ public final class ConnectionSupervisorActor private final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory; @SuppressWarnings("unused") - private ConnectionSupervisorActor(final ActorRef commandForwarderActor, final ActorRef pubSubMediator, - final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory) { + private ConnectionSupervisorActor(final ActorRef commandForwarderActor, + final ActorRef pubSubMediator, + final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory, + final MongoReadJournal mongoReadJournal) { - super(null, CONNECTIVITY_DEFAULT_LOCAL_ASK_TIMEOUT); + super(null, mongoReadJournal); this.commandForwarderActor = commandForwarderActor; this.pubSubMediator = pubSubMediator; this.enforcerActorPropsFactory = enforcerActorPropsFactory; @@ -114,14 +112,16 @@ private ConnectionSupervisorActor(final ActorRef commandForwarderActor, final Ac * @param commandForwarder the actor used to send signals into the ditto cluster. * @param pubSubMediator pub-sub-mediator for the shutdown behavior. * @param enforcerActorPropsFactory used to create the enforcer actor. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the connection. * @return the {@link Props} to create this actor. */ public static Props props(final ActorRef commandForwarder, final ActorRef pubSubMediator, - final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory) { + final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory, + final MongoReadJournal mongoReadJournal) { return Props.create(ConnectionSupervisorActor.class, commandForwarder, pubSubMediator, - enforcerActorPropsFactory); + enforcerActorPropsFactory, mongoReadJournal); } @Override @@ -183,7 +183,7 @@ protected void handleMessagesDuringStartup(final Object message) { @Override protected Props getPersistenceActorProps(final ConnectionId entityId) { - return ConnectionPersistenceActor.props(entityId, commandForwarderActor, pubSubMediator, + return ConnectionPersistenceActor.props(entityId, mongoReadJournal, commandForwarderActor, pubSubMediator, connectivityConfigOverwrites); } @@ -200,6 +200,14 @@ protected ExponentialBackOffConfig getExponentialBackOffConfig() { return connectionConfig.getSupervisorConfig().getExponentialBackOffConfig(); } + @Override + protected LocalAskTimeoutConfig getLocalAskTimeoutConfig() { + return DittoConnectivityConfig.of(DefaultScopedConfig.dittoScoped(getContext().getSystem().settings().config())) + .getConnectionConfig() + .getSupervisorConfig() + .getLocalAskTimeoutConfig(); + } + @Override protected ShutdownBehaviour getShutdownBehaviour(final ConnectionId entityId) { return ShutdownBehaviour.fromIdWithoutNamespace(entityId, pubSubMediator, getSelf()); diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectivityMongoEventAdapter.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectivityMongoEventAdapter.java index b9478f4fd4e..2d1f1465293 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectivityMongoEventAdapter.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectivityMongoEventAdapter.java @@ -12,28 +12,31 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence; +import java.util.HashMap; +import java.util.Map; + import akka.actor.ExtendedActorSystem; import org.eclipse.ditto.base.model.signals.JsonParsable; +import org.eclipse.ditto.base.model.signals.events.Event; import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; import org.eclipse.ditto.base.model.signals.events.EventRegistry; import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry; +import org.eclipse.ditto.base.service.config.DittoServiceConfig; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionModified; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; -import org.eclipse.ditto.connectivity.service.config.DittoConnectivityConfig; -import org.eclipse.ditto.connectivity.service.config.FieldsEncryptionConfig; +import org.eclipse.ditto.connectivity.service.config.DefaultConnectionConfig; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; +import org.eclipse.ditto.connectivity.service.config.DittoConnectivityConfig; +import org.eclipse.ditto.connectivity.service.config.FieldsEncryptionConfig; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; -import java.util.HashMap; -import java.util.Map; - /** * EventAdapter for {@link ConnectivityEvent}s persisted into * akka-persistence event-journal. Converts Events to MongoDB BSON objects and vice versa. @@ -43,8 +46,10 @@ public final class ConnectivityMongoEventAdapter extends AbstractMongoEventAdapt private final FieldsEncryptionConfig encryptionConfig; private final Logger logger; - public ConnectivityMongoEventAdapter(@Nullable final ExtendedActorSystem system) { - super(system, createEventRegistry()); + public ConnectivityMongoEventAdapter(final ExtendedActorSystem system) { + super(system, createEventRegistry(), DefaultConnectionConfig.of( + DittoServiceConfig.of(DefaultScopedConfig.dittoScoped(system.settings().config()), "connectivity")) + .getEventConfig()); logger = LoggerFactory.getLogger(ConnectivityMongoEventAdapter.class); final DittoConnectivityConfig connectivityConfig = DittoConnectivityConfig.of( DefaultScopedConfig.dittoScoped(system.settings().config())); @@ -56,11 +61,14 @@ public ConnectivityMongoEventAdapter(@Nullable final ExtendedActorSystem system) } @Override - protected JsonObject performToJournalMigration(final JsonObject jsonObject) { + protected JsonObjectBuilder performToJournalMigration(final Event event, final JsonObject jsonObject) { if (encryptionConfig.isEncryptionEnabled()) { - return JsonFieldsEncryptor.encrypt(jsonObject, ConnectivityConstants.ENTITY_TYPE.toString(), encryptionConfig.getJsonPointers(), encryptionConfig.getSymmetricalKey()); + final JsonObject superObject = super.performToJournalMigration(event, jsonObject).build(); + return JsonFieldsEncryptor.encrypt(superObject, ConnectivityConstants.ENTITY_TYPE.toString(), + encryptionConfig.getJsonPointers(), encryptionConfig.getSymmetricalKey()) + .toBuilder(); } - return jsonObject; + return super.performToJournalMigration(event, jsonObject); } @Override diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/JsonFieldsEncryptor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/JsonFieldsEncryptor.java index a1d8a9ac4f6..fc617f226e8 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/JsonFieldsEncryptor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/JsonFieldsEncryptor.java @@ -87,7 +87,7 @@ public static JsonObject decrypt(final JsonObject jsonObject, final String point } static String replaceUriPassword(final String uriStringRepresentation, final String patchedPassword) { - final String userInfo = URI.create(uriStringRepresentation).getUserInfo(); + final String userInfo = URI.create(uriStringRepresentation).getRawUserInfo(); final String newUserInfo = userInfo.substring(0, userInfo.indexOf(":") + 1) + patchedPassword; final int startOfPwd = uriStringRepresentation.indexOf(userInfo); final int endOfPassword = uriStringRepresentation.indexOf("@"); @@ -159,7 +159,7 @@ private static Optional getUriPassword(final String uriStringRepresentat .message("Not a valid connection URI") .build(); } - final String userInfo = uri.getUserInfo(); + final String userInfo = uri.getRawUserInfo(); if (userInfo == null) { return Optional.empty(); } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/stages/StagedCommand.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/stages/StagedCommand.java index 96777760a5a..140dbb0501f 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/stages/StagedCommand.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/stages/StagedCommand.java @@ -27,6 +27,8 @@ import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.WithConnectionId; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.json.JsonField; @@ -41,7 +43,8 @@ * It contains a sequence of actions. Some actions are asynchronous. The connection actor can thus schedule the next * action as a staged command to self after an asynchronous action. Synchronous actions can be executed right away. */ -public final class StagedCommand implements ConnectivityCommand, Iterator { +public final class StagedCommand implements ConnectivityCommand, Iterator, + WithConnectionId { private final ConnectivityCommand command; @Nullable private final ConnectivityEvent event; @@ -82,6 +85,17 @@ public ConnectivityCommand getCommand() { return command; } + @Override + public ConnectionId getEntityId() { + if (command instanceof WithConnectionId withConnectionId) { + return withConnectionId.getEntityId(); + } else if (event != null) { + return event.getEntityId(); + } else { + throw new IllegalStateException("Could not determine ConnectionId in StagedCommand"); + } + } + /** * @return the event to persist, apply or publish or dummy-event. */ @@ -210,4 +224,5 @@ public ConnectionAction nextAction() { private Queue getActionsAsQueue() { return new LinkedList<>(actions); } + } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/AbstractConnectivityCommandStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/AbstractConnectivityCommandStrategy.java index 2d11f67ebbf..65608619c5e 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/AbstractConnectivityCommandStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/AbstractConnectivityCommandStrategy.java @@ -29,7 +29,8 @@ import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.commands.AbstractCommandStrategy; +import org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator; +import org.eclipse.ditto.internal.utils.persistentactors.etags.AbstractConditionHeaderCheckingCommandStrategy; /** * Abstract base class for {@link org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand} strategies. @@ -37,12 +38,20 @@ * @param the type of the handled command */ abstract class AbstractConnectivityCommandStrategy> - extends AbstractCommandStrategy> { + extends AbstractConditionHeaderCheckingCommandStrategy> { - AbstractConnectivityCommandStrategy(final Class theMatchingClass) { + private static final ConditionalHeadersValidator VALIDATOR = + ConnectionsConditionalHeadersValidatorProvider.getInstance(); + + protected AbstractConnectivityCommandStrategy(final Class theMatchingClass) { super(theMatchingClass); } + @Override + protected ConditionalHeadersValidator getValidator() { + return VALIDATOR; + } + @Override public boolean isDefined(final C command) { return command instanceof ConnectivityCommand || command instanceof ConnectivitySudoCommand; diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CloseConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CloseConnectionStrategy.java index 3b3bb0e322e..36db15991b5 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CloseConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CloseConnectionStrategy.java @@ -16,20 +16,22 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CloseConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CloseConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionClosed; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.CloseConnection} command. @@ -56,4 +58,15 @@ protected Result> doApply(final Context co ConnectionAction.SEND_RESPONSE); return newMutationResult(StagedCommand.of(command, event, response, actions), event, response); } + + @Override + public Optional previousEntityTag(final CloseConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final CloseConnection command, @Nullable final Connection newEntity) { + return Optional.of(getEntityOrThrow(newEntity)).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionConflictStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionConflictStrategy.java index 8387721afdc..9d6050e93fc 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionConflictStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionConflictStrategy.java @@ -14,15 +14,18 @@ import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newErrorResult; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionConflictException; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnection; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnection} command @@ -49,4 +52,15 @@ protected Result> doApply(final Context co .build(); return newErrorResult(conflictException, command); } + + @Override + public Optional previousEntityTag(final CreateConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final CreateConnection command, @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionUninitializedStrategies.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionUninitializedStrategies.java deleted file mode 100644 index f606c1e0215..00000000000 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionUninitializedStrategies.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2021 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ -package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; - -import java.util.Optional; -import java.util.function.Consumer; - -import javax.annotation.Nullable; - -import org.eclipse.ditto.base.model.entity.metadata.Metadata; -import org.eclipse.ditto.base.model.signals.commands.Command; -import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; -import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.commands.AbstractCommandStrategy; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; -import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; - -/** - * Strategies to handle signals as an uninitialized connection that stashes every command. - */ -public final class ConnectionUninitializedStrategies - extends AbstractCommandStrategy, Connection, ConnectionState, ConnectivityEvent> - implements ConnectivityCommandStrategies { - - private final Consumer> action; - - private ConnectionUninitializedStrategies(final Consumer> action) { - super(Command.class); - this.action = action; - } - - /** - * Return a new instance of this class. - * - * @param action what to do on connectivity commands. - * @return the empty result. - */ - public static ConnectionUninitializedStrategies of(final Consumer> action) { - return new ConnectionUninitializedStrategies(action); - } - - @Override - public boolean isDefined(final Command command) { - return true; - } - - @Override - protected Optional calculateRelativeMetadata(@Nullable final Connection entity, - final Command command) { - return Optional.empty(); - } - - @Override - protected Result> doApply(final Context context, - @Nullable final Connection entity, - final long nextRevision, final Command command, @Nullable final Metadata metadata) { - action.accept(command); - return Result.empty(); - } - - @Override - public Result> unhandled(final Context context, - @Nullable final Connection entity, - final long nextRevision, - final Command command) { - - return ResultFactory.newErrorResult(ConnectionNotAccessibleException - .newBuilder(context.getState().id()) - .dittoHeaders(command.getDittoHeaders()) - .build(), command); - } - -} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionsConditionalHeadersValidatorProvider.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionsConditionalHeadersValidatorProvider.java new file mode 100644 index 00000000000..f4dcca5b163 --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionsConditionalHeadersValidatorProvider.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionPreconditionFailedException; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionPreconditionNotModifiedException; +import org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator; + +/** + * Provides a {@link ConditionalHeadersValidator} which checks conditional (http) headers based on a given ETag on + * Connection resources. + */ +@Immutable +final class ConnectionsConditionalHeadersValidatorProvider { + + /** + * Settings for validating conditional headers on Connection resources. + */ + private static class ConnectionsConditionalHeadersValidationSettings + implements ConditionalHeadersValidator.ValidationSettings { + + /** + * Returns a builder for a {@link org.eclipse.ditto.things.model.signals.commands.exceptions.ThingPreconditionFailedException}. + * + * @param conditionalHeaderName the name of the conditional header. + * @param expected the expected value. + * @param actual the actual ETag value. + * @return the builder. + */ + @Override + public DittoRuntimeExceptionBuilder createPreconditionFailedExceptionBuilder(final String conditionalHeaderName, + final String expected, final String actual) { + return ConnectionPreconditionFailedException.newBuilder(conditionalHeaderName, expected, actual); + } + + /** + * Returns a builder for a {@link ConnectionPreconditionNotModifiedException}. + * + * @param expectedNotToMatch the value which was expected not to match {@code matched} value. + * @param matched the matched value. + * @return the builder. + */ + @Override + public DittoRuntimeExceptionBuilder createPreconditionNotModifiedExceptionBuilder( + final String expectedNotToMatch, final String matched) { + return ConnectionPreconditionNotModifiedException.newBuilder(expectedNotToMatch, matched); + } + + @Override + public DittoRuntimeExceptionBuilder createPreconditionNotModifiedForEqualityExceptionBuilder() { + return ConnectionPreconditionNotModifiedException.newBuilder() + .message("The previous value was equal to the new value and the 'if-equal' header was set to 'skip'.") + .description("Your changes were not applied, which is probably the expected outcome."); + } + } + + private static final ConditionalHeadersValidator INSTANCE = createInstance(); + + private ConnectionsConditionalHeadersValidatorProvider() { + throw new AssertionError(); + } + + /** + * Returns the (singleton) instance of {@link org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator} for Thing resources. + * + * @return the {@link org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator}. + */ + public static ConditionalHeadersValidator getInstance() { + return INSTANCE; + } + + private static ConditionalHeadersValidator createInstance() { + return ConditionalHeadersValidator.of(new ConnectionsConditionalHeadersValidationSettings(), + checker -> false); + } + +} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CreateConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CreateConnectionStrategy.java index e4978b5a5e0..0e640ab3cc0 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CreateConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CreateConnectionStrategy.java @@ -22,7 +22,9 @@ import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newErrorResult; import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newMutationResult; +import java.time.Instant; import java.util.List; +import java.util.Objects; import java.util.Optional; import javax.annotation.Nullable; @@ -30,16 +32,17 @@ import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityStatus; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link CreateConnection} command. @@ -50,6 +53,21 @@ final class CreateConnectionStrategy extends AbstractConnectivityCommandStrategy super(CreateConnection.class); } + @Override + public boolean isDefined(final CreateConnection command) { + return true; + } + + @Override + public boolean isDefined(final Context context, @Nullable final Connection connection, + final CreateConnection command) { + final boolean connectionExists = Optional.ofNullable(connection) + .map(t -> !t.isDeleted()) + .orElse(false); + + return !connectionExists && Objects.equals(context.getState().id(), command.getEntityId()); + } + @Override protected Result> doApply(final Context context, @Nullable final Connection entity, @@ -57,7 +75,12 @@ protected Result> doApply(final Context co final CreateConnection command, @Nullable final Metadata metadata) { - final Connection connection = command.getConnection().toBuilder().lifecycle(ACTIVE).build(); + final Instant timestamp = getEventTimestamp(); + final Connection connection = command.getConnection().toBuilder().lifecycle(ACTIVE) + .revision(nextRevision) + .created(timestamp) + .modified(timestamp) + .build(); final ConnectivityEvent event = ConnectionCreated.of(connection, nextRevision, getEventTimestamp(), command.getDittoHeaders(), metadata); @@ -78,4 +101,15 @@ protected Result> doApply(final Context co return newMutationResult(command, event, response, true, false); } } + + @Override + public Optional previousEntityTag(final CreateConnection command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final CreateConnection command, @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/DeleteConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/DeleteConnectionStrategy.java index f27f0559c68..459aed74e2d 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/DeleteConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/DeleteConnectionStrategy.java @@ -16,23 +16,25 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.DeleteConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.DeleteConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionDeleted; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.DeleteConnection} + * This strategy handles the {@link DeleteConnection} * command. */ final class DeleteConnectionStrategy extends AbstractConnectivityCommandStrategy { @@ -61,4 +63,14 @@ protected Result> doApply(final Context co return newMutationResult(StagedCommand.of(command, event, response, actions), event, response); } + @Override + public Optional previousEntityTag(final DeleteConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final DeleteConnection command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/EnableConnectionLogsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/EnableConnectionLogsStrategy.java index b6047eaebbd..5f2c17b9fe2 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/EnableConnectionLogsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/EnableConnectionLogsStrategy.java @@ -14,16 +14,21 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; + +import javax.annotation.Nullable; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.EnableConnectionLogs; import org.eclipse.ditto.connectivity.model.signals.commands.modify.EnableConnectionLogsResponse; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.EnableConnectionLogs} command. + * This strategy handles the {@link EnableConnectionLogs} command. */ final class EnableConnectionLogsStrategy extends AbstractEphemeralStrategy { @@ -40,4 +45,15 @@ WithDittoHeaders getResponse(final ConnectionState state, final DittoHeaders hea List getActions() { return Arrays.asList(ConnectionAction.BROADCAST_TO_CLIENT_ACTORS_IF_STARTED, ConnectionAction.SEND_RESPONSE, ConnectionAction.ENABLE_LOGGING); } + + @Override + public Optional previousEntityTag(final EnableConnectionLogs command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final EnableConnectionLogs command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/LoggingExpiredStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/LoggingExpiredStrategy.java index 81f09893640..e44ccc0b1f5 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/LoggingExpiredStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/LoggingExpiredStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import java.util.Optional; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.LoggingExpired; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.LoggingExpired} command. + * This strategy handles the {@link LoggingExpired} command. */ final class LoggingExpiredStrategy extends AbstractSingleActionStrategy { @@ -28,4 +34,15 @@ final class LoggingExpiredStrategy extends AbstractSingleActionStrategy previousEntityTag(final LoggingExpired command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final LoggingExpired command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ModifyConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ModifyConnectionStrategy.java index d8252b31f1b..2c9813c8a49 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ModifyConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ModifyConnectionStrategy.java @@ -16,6 +16,7 @@ import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newErrorResult; import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newMutationResult; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -25,21 +26,22 @@ import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionConfigurationInvalidException; import org.eclipse.ditto.connectivity.model.ConnectivityStatus; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnectionResponse; +import org.eclipse.ditto.connectivity.model.signals.events.ConnectionModified; +import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; -import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; -import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnectionResponse; -import org.eclipse.ditto.connectivity.model.signals.events.ConnectionModified; -import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection} command. + * This strategy handles the {@link ModifyConnection} command. */ final class ModifyConnectionStrategy extends AbstractConnectivityCommandStrategy { @@ -54,7 +56,11 @@ protected Result> doApply(final Context co final ModifyConnection command, @Nullable final Metadata metadata) { - final Connection connection = command.getConnection().toBuilder().lifecycle(ACTIVE).build(); + final Instant eventTs = getEventTimestamp(); + final Connection connection = command.getConnection().toBuilder().lifecycle(ACTIVE) + .revision(nextRevision) + .modified(eventTs) + .build(); if (entity != null && entity.getConnectionType() != connection.getConnectionType() && !command.getDittoHeaders().isSudo()) { return ResultFactory.newErrorResult( @@ -96,4 +102,15 @@ protected Result> doApply(final Context co return newMutationResult(command, event, response); } } + + @Override + public Optional previousEntityTag(final ModifyConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final ModifyConnection command, @Nullable final Connection newEntity) { + return Optional.of(getEntityOrThrow(newEntity)).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/OpenConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/OpenConnectionStrategy.java index f5978d4ec57..14048528575 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/OpenConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/OpenConnectionStrategy.java @@ -29,15 +29,16 @@ import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.OpenConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.OpenConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionOpened; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link OpenConnection} command. @@ -69,4 +70,15 @@ protected Result> doApply(final Context co return newMutationResult(StagedCommand.of(command, event, response, actions), event, response); } } + + @Override + public Optional previousEntityTag(final OpenConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final OpenConnection command, @Nullable final Connection newEntity) { + return Optional.of(getEntityOrThrow(newEntity)).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionLogsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionLogsStrategy.java index aa80d590d8a..fc59fe58805 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionLogsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionLogsStrategy.java @@ -14,13 +14,18 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; + +import javax.annotation.Nullable; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionLogs; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionLogsResponse; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionLogs} @@ -41,4 +46,15 @@ WithDittoHeaders getResponse(final ConnectionState state, final DittoHeaders hea List getActions() { return Arrays.asList(ConnectionAction.BROADCAST_TO_CLIENT_ACTORS_IF_STARTED, ConnectionAction.SEND_RESPONSE); } + + @Override + public Optional previousEntityTag(final ResetConnectionLogs command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final ResetConnectionLogs command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionMetricsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionMetricsStrategy.java index 40db31d9de2..f7a3faf7625 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionMetricsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionMetricsStrategy.java @@ -14,16 +14,21 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; + +import javax.annotation.Nullable; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionMetrics; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionMetricsResponse; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionMetrics} + * This strategy handles the {@link ResetConnectionMetrics} * command. */ final class ResetConnectionMetricsStrategy extends AbstractEphemeralStrategy { @@ -41,4 +46,16 @@ WithDittoHeaders getResponse(final ConnectionState state, final DittoHeaders hea List getActions() { return Arrays.asList(ConnectionAction.BROADCAST_TO_CLIENT_ACTORS_IF_STARTED, ConnectionAction.SEND_RESPONSE); } + + @Override + public Optional previousEntityTag(final ResetConnectionMetrics command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final ResetConnectionMetrics command, + @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionLogsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionLogsStrategy.java index c93539703cf..988b39526cc 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionLogsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionLogsStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import java.util.Optional; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionLogs; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionLogs} + * This strategy handles the {@link RetrieveConnectionLogs} * command. */ final class RetrieveConnectionLogsStrategy extends AbstractSingleActionStrategy { @@ -29,4 +35,16 @@ final class RetrieveConnectionLogsStrategy extends AbstractSingleActionStrategy< ConnectionAction getAction() { return ConnectionAction.RETRIEVE_CONNECTION_LOGS; } + + @Override + public Optional previousEntityTag(final RetrieveConnectionLogs command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveConnectionLogs command, + @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionMetricsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionMetricsStrategy.java index a6de954cfb6..c9f801253a1 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionMetricsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionMetricsStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import java.util.Optional; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionMetrics; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionMetrics} + * This strategy handles the {@link RetrieveConnectionMetrics} * command. */ final class RetrieveConnectionMetricsStrategy extends AbstractSingleActionStrategy { @@ -29,4 +35,16 @@ final class RetrieveConnectionMetricsStrategy extends AbstractSingleActionStrate ConnectionAction getAction() { return ConnectionAction.RETRIEVE_CONNECTION_METRICS; } + + @Override + public Optional previousEntityTag(final RetrieveConnectionMetrics command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveConnectionMetrics command, + @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStatusStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStatusStrategy.java index 98bba007344..17d63c06cf3 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStatusStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStatusStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import java.util.Optional; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionStatus; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionStatus} + * This strategy handles the {@link RetrieveConnectionStatus} * command. */ final class RetrieveConnectionStatusStrategy extends AbstractSingleActionStrategy { @@ -29,4 +35,16 @@ final class RetrieveConnectionStatusStrategy extends AbstractSingleActionStrateg ConnectionAction getAction() { return ConnectionAction.RETRIEVE_CONNECTION_STATUS; } + + @Override + public Optional previousEntityTag(final RetrieveConnectionStatus command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveConnectionStatus command, + @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStrategy.java index 0efc54c8747..8afe2019b92 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStrategy.java @@ -12,19 +12,26 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.DittoHeadersSettable; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; -import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; +import org.eclipse.ditto.connectivity.model.signals.commands.query.ConnectivityQueryCommand; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; +import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; +import org.eclipse.ditto.json.JsonObject; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection} command. + * This strategy handles the {@link RetrieveConnection} command. */ final class RetrieveConnectionStrategy extends AbstractConnectivityCommandStrategy { @@ -41,9 +48,40 @@ protected Result> doApply(final Context co if (entity != null) { return ResultFactory.newQueryResult(command, - RetrieveConnectionResponse.of(entity.toJson(), command.getDittoHeaders())); + appendETagHeaderIfProvided(command, getRetrieveConnectionResponse(entity, command), entity) + ); } else { return ResultFactory.newErrorResult(notAccessible(context, command), command); } } + + @Override + public Optional previousEntityTag(final RetrieveConnection command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveConnection command, @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } + + private static DittoHeadersSettable getRetrieveConnectionResponse(@Nullable final Connection connection, + final ConnectivityQueryCommand command) { + if (connection != null) { + return RetrieveConnectionResponse.of(getConnectionJson(connection, command), + command.getDittoHeaders()); + } else { + return ConnectionNotAccessibleException.newBuilder(((RetrieveConnection) command).getEntityId()) + .dittoHeaders(command.getDittoHeaders()) + .build(); + } + } + + private static JsonObject getConnectionJson(final Connection connection, + final ConnectivityQueryCommand command) { + return ((RetrieveConnection) command).getSelectedFields() + .map(selectedFields -> connection.toJson(command.getImplementedSchemaVersion(), selectedFields)) + .orElseGet(() -> connection.toJson(command.getImplementedSchemaVersion())); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveResolvedHonoConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveResolvedHonoConnectionStrategy.java index d997d23cb4f..aef990c5d4c 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveResolvedHonoConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveResolvedHonoConnectionStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.DittoHeadersSettable; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionType; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; +import org.eclipse.ditto.connectivity.model.signals.commands.query.ConnectivityQueryCommand; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveResolvedHonoConnection; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; @@ -24,6 +30,7 @@ import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; +import org.eclipse.ditto.json.JsonObject; import akka.actor.ActorSystem; @@ -49,13 +56,45 @@ protected Result> doApply(final Context co final Result> result; if (entity != null && entity.getConnectionType() == ConnectionType.HONO) { - final var json = honoConnectionFactory.getHonoConnection(entity).toJson(); - - result = ResultFactory.newQueryResult(command, - RetrieveConnectionResponse.of(json, command.getDittoHeaders())); + return ResultFactory.newQueryResult(command, + appendETagHeaderIfProvided(command, getRetrieveConnectionResponse(entity, command), entity) + ); } else { result = ResultFactory.newErrorResult(notAccessible(context, command), command); } return result; } + + @Override + public Optional previousEntityTag(final RetrieveResolvedHonoConnection command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveResolvedHonoConnection command, + @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } + + private DittoHeadersSettable getRetrieveConnectionResponse(@Nullable final Connection connection, + final ConnectivityQueryCommand command) { + if (connection != null) { + return RetrieveConnectionResponse.of(getConnectionJson(connection, command), + command.getDittoHeaders()); + } else { + return ConnectionNotAccessibleException.newBuilder(((RetrieveResolvedHonoConnection) command).getEntityId()) + .dittoHeaders(command.getDittoHeaders()) + .build(); + } + } + + private JsonObject getConnectionJson(final Connection connection, + final ConnectivityQueryCommand command) { + + final Connection honoConnection = honoConnectionFactory.getHonoConnection(connection); + return ((RetrieveResolvedHonoConnection) command).getSelectedFields() + .map(selectedFields -> honoConnection.toJson(command.getImplementedSchemaVersion(), selectedFields)) + .orElseGet(() -> honoConnection.toJson(command.getImplementedSchemaVersion())); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/StagedCommandStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/StagedCommandStrategy.java index b35efef2929..50921897e83 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/StagedCommandStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/StagedCommandStrategy.java @@ -13,17 +13,19 @@ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; import java.text.MessageFormat; +import java.util.Optional; import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityInternalErrorException; +import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; -import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand} @@ -50,4 +52,15 @@ protected Result> doApply(final Context co .dittoHeaders(command.getDittoHeaders()) .build(), command)); } + + @Override + public Optional previousEntityTag(final StagedCommand command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final StagedCommand command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoAddConnectionLogEntryStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoAddConnectionLogEntryStrategy.java index 9f71bd93e8b..3c59a182321 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoAddConnectionLogEntryStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoAddConnectionLogEntryStrategy.java @@ -12,9 +12,12 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoAddConnectionLogEntry; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionId; @@ -58,4 +61,16 @@ private ConnectionLogger getAppropriateLogger(final ConnectionLoggerRegistry con logEntry.getLogType(), logEntry.getAddress().orElse(null)); } + + @Override + public Optional previousEntityTag(final SudoAddConnectionLogEntry command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final SudoAddConnectionLogEntry command, + @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoRetrieveConnectionTagsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoRetrieveConnectionTagsStrategy.java index 0b8b7a30fa9..f447b56f18e 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoRetrieveConnectionTagsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoRetrieveConnectionTagsStrategy.java @@ -12,9 +12,12 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionTags; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionTagsResponse; import org.eclipse.ditto.connectivity.model.Connection; @@ -46,4 +49,16 @@ protected Result> doApply(final Context co return ResultFactory.newErrorResult(notAccessible(context, command), command); } } + + @Override + public Optional previousEntityTag(final SudoRetrieveConnectionTags command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final SudoRetrieveConnectionTags command, + @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionConflictStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionConflictStrategy.java index 6837d04401d..7ce4217df36 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionConflictStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionConflictStrategy.java @@ -14,15 +14,18 @@ import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newQueryResult; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnection} command @@ -44,4 +47,15 @@ protected Result> doApply(final Context co return newQueryResult(command, TestConnectionResponse.alreadyCreated(context.getState().id(), command.getDittoHeaders())); } + + @Override + public Optional previousEntityTag(final TestConnection command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final TestConnection command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionStrategy.java index 42ae1cc3514..4001ceb4f75 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionStrategy.java @@ -18,21 +18,23 @@ import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnection} command. @@ -43,6 +45,20 @@ final class TestConnectionStrategy extends AbstractConnectivityCommandStrategy context, @Nullable final Connection connection, final TestConnection command) { + final boolean connectionExists = Optional.ofNullable(connection) + .map(t -> !t.isDeleted()) + .orElse(false); + + return !connectionExists && Objects.equals(context.getState().id(), command.getEntityId()); + } + @Override protected Result> doApply(final Context context, @Nullable final Connection entity, @@ -67,4 +83,15 @@ protected Result> doApply(final Context co TestConnectionResponse.alreadyCreated(context.getState().id(), command.getDittoHeaders())); } } + + @Override + public Optional previousEntityTag(final TestConnection command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final TestConnection command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionClosedStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionClosedStrategy.java index f106c7b3c4d..cc65c9f2dfc 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionClosedStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionClosedStrategy.java @@ -18,11 +18,11 @@ import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityStatus; -import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionClosed; +import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.events.ConnectionClosed} event. + * This strategy handles the {@link ConnectionClosed} event. */ final class ConnectionClosedStrategy implements EventStrategy { diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionCreatedStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionCreatedStrategy.java index 10fc23649c7..27cabadce17 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionCreatedStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionCreatedStrategy.java @@ -15,11 +15,11 @@ import javax.annotation.Nullable; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; +import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated} event. + * This strategy handles the {@link ConnectionCreated} event. */ final class ConnectionCreatedStrategy implements EventStrategy { diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionDeletedStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionDeletedStrategy.java index ed958633104..5709cda7baf 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionDeletedStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionDeletedStrategy.java @@ -16,11 +16,11 @@ import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionLifecycle; -import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionDeleted; +import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.events.ConnectionDeleted} event. + * This strategy handles the {@link ConnectionDeleted} event. */ final class ConnectionDeletedStrategy implements EventStrategy { @@ -29,7 +29,9 @@ final class ConnectionDeletedStrategy implements EventStrategy { diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionOpenedStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionOpenedStrategy.java index 517b0986a45..884dbd10742 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionOpenedStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionOpenedStrategy.java @@ -18,17 +18,20 @@ import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityStatus; -import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionOpened; +import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.events.ConnectionOpened} event. + * This strategy handles the {@link ConnectionOpened} event. */ final class ConnectionOpenedStrategy implements EventStrategy { @Override public Connection handle(final ConnectionOpened event, @Nullable final Connection connection, final long revision) { - return checkNotNull(connection, "connection").toBuilder().connectionStatus(ConnectivityStatus.OPEN).build(); + return checkNotNull(connection, "connection") + .toBuilder() + .connectionStatus(ConnectivityStatus.OPEN) + .build(); } } diff --git a/connectivity/service/src/main/resources/connectivity-extension.conf b/connectivity/service/src/main/resources/connectivity-extension.conf index e69de29bb2d..0529d13a7a2 100644 --- a/connectivity/service/src/main/resources/connectivity-extension.conf +++ b/connectivity/service/src/main/resources/connectivity-extension.conf @@ -0,0 +1,5 @@ +ditto.extensions { + signal-transformers-provider.extension-config.signal-transformers = [ + "org.eclipse.ditto.connectivity.service.enforcement.pre.ModifyToCreateConnectionTransformer", // always keep this as first transformer in order to guarantee that all following transformers know that the command is creating a connection instead of modifying it + ] ${ditto.extensions.signal-transformers-provider.extension-config.signal-transformers} +} diff --git a/connectivity/service/src/main/resources/connectivity.conf b/connectivity/service/src/main/resources/connectivity.conf index 6c58dc8e0ee..5191d6e0982 100644 --- a/connectivity/service/src/main/resources/connectivity.conf +++ b/connectivity/service/src/main/resources/connectivity.conf @@ -205,11 +205,33 @@ ditto { random-factor = 1.0 corrupted-receive-timeout = 600s } + + # For connectivity, this local ask timeout has to be higher as e.g. "openConnection" commands performed in a + # "staged" way will lead to quite some response times. + local-ask { + timeout = 50s + timeout = ${?THINGS_SUPERVISOR_LOCAL_ASK_TIMEOUT} + } } snapshot { - threshold = 10 + # the interval when to do snapshot for a Connection which had changes to it interval = 15m + interval = ${?CONNECTION_SNAPSHOT_INTERVAL} # may be overridden with this environment variable + + # the threshold after how many changes to a Connection to do a snapshot + threshold = 10 + threshold = ${?CONNECTION_SNAPSHOT_THRESHOLD} # may be overridden with this environment variable + } + + event { + # define the DittoHeaders to persist when persisting events to the journal + # those can e.g. be retrieved as additional "audit log" information when accessing a historical connection revision + historical-headers-to-persist = [ + #"ditto-originator" # who (user-subject/connection-pre-auth-subject) issued the event + #"correlation-id" + ] + historical-headers-to-persist = ${?CONNECTION_EVENT_HISTORICAL_HEADERS_TO_PERSIST} } activity-check { @@ -677,27 +699,62 @@ ditto { } cleanup { + # enabled configures whether background cleanup is enabled or not + # If enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process enabled = true enabled = ${?CLEANUP_ENABLED} + # history-retention-duration configures the duration of how long to "keep" events and snapshots before being + # allowed to remove them in scope of cleanup. + # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read + # journal. + history-retention-duration = 30d + history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} + + # quiet-period defines how long to stay in a state where the background cleanup is not yet started + # Applies after: + # - starting the service + # - each "completed" background cleanup run (all entities were cleaned up) quiet-period = 5m quiet-period = ${?CLEANUP_QUIET_PERIOD} + # interval configures how often a "credit decision" is made. + # The background cleanup works with a credit system and does only generate new "cleanup credits" if the MongoDB + # currently has capacity to do cleanups. interval = 60s interval = ${?CLEANUP_INTERVAL} + # timer-threshold configures the maximum database latency to give out credit for cleanup actions. + # If write operations to the MongoDB within the last `interval` had a `max` value greater to the configured + # threshold, no new cleanup credits will be issued for the next `interval`. + # Which throttles cleanup when MongoDB is currently under heavy (write) load. timer-threshold = 150ms timer-threshold = ${?CLEANUP_TIMER_THRESHOLD} + # credits-per-batch configures how many "cleanup credits" should be generated per `interval` as long as the + # write operations to the MongoDB are less than the configured `timer-threshold`. + # Limits the rate of cleanup actions to this many per credit decision interval. + # One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`. credits-per-batch = 1 credits-per-batch = ${?CLEANUP_CREDITS_PER_BATCH} + # reads-per-query configures the number of snapshots to scan per MongoDB query. + # Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned + # with the amount of "cleanup credits" issued per `interval` - in order to avoid long running queries. reads-per-query = 100 reads-per-query = ${?CLEANUP_READS_PER_QUERY} + # writes-per-credit configures the number of documents to delete for each credit. + # If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead + # to 10 delete operations performed against MongoDB. writes-per-credit = 100 writes-per-credit = ${?CLEANUP_WRITES_PER_CREDIT} + # delete-final-deleted-snapshot configures whether for a deleted entity, the final snapshot (containing the + # "deleted" information) should be deleted or not. + # If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with + # a revision number 1 higher than the previously deleted entity. If the final snapshot is deleted as well, + # recreation of an entity with the same ID will lead to revisionNumber=1 after its recreation. delete-final-deleted-snapshot = false delete-final-deleted-snapshot = ${?CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT} } @@ -1113,9 +1170,31 @@ akka-contrib-mongodb-persistence-connection-journal { } } +akka-contrib-mongodb-persistence-connection-journal-read { + class = "akka.contrib.persistence.mongodb.MongoReadJournal" + plugin-dispatcher = "connection-persistence-dispatcher" + + overrides { + journal-collection = "connection_journal" + journal-index = "connection_journal_index" + realtime-collection = "connection_realtime" + metadata-collection = "connection_metadata" + } +} + akka-contrib-mongodb-persistence-connection-snapshots { class = "akka.contrib.persistence.mongodb.MongoSnapshots" plugin-dispatcher = "connection-persistence-dispatcher" + + circuit-breaker { + max-failures = 5 # if an exception during persisting an event/snapshot occurs this often -- a successful write resets the counter + max-failures = ${?SNAPSHOT_BREAKER_MAXTRIES} + call-timeout = 10s # MongoDB Timeouts causing the circuitBreaker to open + call-timeout = ${?SNAPSHOT_BREAKER_TIMEOUT} + reset-timeout = 6s # after this time in "Open" state, the cicuitBreaker is "Half-opened" again + reset-timeout = ${?SNAPSHOT_BREAKER_RESET} + } + overrides { snaps-collection = "connection_snaps" snaps-index = "connection_snaps_index" diff --git a/connectivity/service/src/main/resources/javascript/incoming-mapping.js b/connectivity/service/src/main/resources/javascript/incoming-mapping.js index d85f3c61cbf..b9f6beeda01 100644 --- a/connectivity/service/src/main/resources/javascript/incoming-mapping.js +++ b/connectivity/service/src/main/resources/javascript/incoming-mapping.js @@ -19,7 +19,7 @@ function mapToDittoProtocolMsg( // ### Insert/adapt your mapping logic here. // Use helper function Ditto.buildDittoProtocolMsg to build Ditto protocol message // based on incoming payload. - // See https://www.eclipse.org/ditto/connectivity-mapping.html#helper-functions for details. + // See https://www.eclipse.dev/ditto/connectivity-mapping.html#helper-functions for details. // ### example code assuming the Ditto protocol content type for incoming messages. if (contentType === 'application/vnd.eclipse.ditto+json') { diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalCommandRegistryTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalCommandRegistryTest.java index 8feb00887e4..827d15af927 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalCommandRegistryTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalCommandRegistryTest.java @@ -17,6 +17,7 @@ import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoAddConnectionLogEntry; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionIdsByTag; @@ -66,7 +67,8 @@ public ConnectivityServiceGlobalCommandRegistryTest() { PurgeEntities.class, ModifySplitBrainResolver.class, PublishSignal.class, - SudoAddConnectionLogEntry.class + SudoAddConnectionLogEntry.class, + SubscribeForPersistedEvents.class ); } diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalEventRegistryTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalEventRegistryTest.java index 4df61bcefb2..ae6b9142a42 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalEventRegistryTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalEventRegistryTest.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.connectivity.service; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.internal.utils.persistentactors.EmptyEvent; import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases; @@ -31,7 +32,8 @@ public ConnectivityServiceGlobalEventRegistryTest() { SubscriptionCreated.class, ThingsOutOfSync.class, ThingSnapshotTaken.class, - EmptyEvent.class + EmptyEvent.class, + StreamingSubscriptionComplete.class ); } diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/ErrorHandlingActorTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/ErrorHandlingActorTest.java index 7391eb710c3..d40dd99cad9 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/ErrorHandlingActorTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/ErrorHandlingActorTest.java @@ -16,6 +16,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import org.assertj.core.api.Assertions; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionId; @@ -91,7 +92,12 @@ public void tryCreateConnectionExpectSuccessResponseIndependentOfConnectionStatu // create connection final ConnectivityModifyCommand command = CreateConnection.of(connection, DittoHeaders.empty()); underTest.tell(command, getRef()); - expectMsg(CreateConnectionResponse.of(connection, DittoHeaders.empty())); + final CreateConnectionResponse resp = + expectMsgClass(dilated(CONNECT_TIMEOUT), CreateConnectionResponse.class); + Assertions.assertThat(resp.getConnection()) + .usingRecursiveComparison() + .ignoringFields("revision", "modified", "created") + .isEqualTo(connection); }}; tearDown(); } @@ -121,9 +127,12 @@ public void tryDeleteConnectionExpectErrorResponse() { // create connection final CreateConnection createConnection = CreateConnection.of(connection, DittoHeaders.empty()); underTest.tell(createConnection, getRef()); - final CreateConnectionResponse createConnectionResponse = - CreateConnectionResponse.of(connection, DittoHeaders.empty()); - expectMsg(dilated(CONNECT_TIMEOUT), createConnectionResponse); + final CreateConnectionResponse resp = + expectMsgClass(dilated(CONNECT_TIMEOUT), CreateConnectionResponse.class); + Assertions.assertThat(resp.getConnection()) + .usingRecursiveComparison() + .ignoringFields("revision", "modified", "created") + .isEqualTo(connection); // delete connection final ConnectivityModifyCommand command = DeleteConnection.of(connectionId, DittoHeaders.empty()); @@ -147,9 +156,12 @@ private void tryModifyConnectionExpectErrorResponse(final String action) { // create connection final CreateConnection createConnection = CreateConnection.of(connection, DittoHeaders.empty()); underTest.tell(createConnection, getRef()); - final CreateConnectionResponse createConnectionResponse = - CreateConnectionResponse.of(connection, DittoHeaders.empty()); - expectMsg(createConnectionResponse); + final CreateConnectionResponse resp = + expectMsgClass(dilated(CONNECT_TIMEOUT), CreateConnectionResponse.class); + Assertions.assertThat(resp.getConnection()) + .usingRecursiveComparison() + .ignoringFields("revision", "modified", "created") + .isEqualTo(connection); // modify connection final ConnectivityModifyCommand command; diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/OutboundMappingProcessorActorTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/OutboundMappingProcessorActorTest.java index 7009ed6ff62..3ec16166e1c 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/OutboundMappingProcessorActorTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/OutboundMappingProcessorActorTest.java @@ -63,11 +63,11 @@ import akka.testkit.TestProbe; import akka.testkit.javadsl.TestKit; -@FixMethodOrder(MethodSorters.DEFAULT) /** * Tests in addition to {@link MessageMappingProcessorActorTest} * for {@link OutboundMappingProcessorActor} only. */ +@FixMethodOrder(MethodSorters.DEFAULT) public final class OutboundMappingProcessorActorTest { @ClassRule diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/TestConstants.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/TestConstants.java index b0a787054d4..da6dcfb5ea6 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/TestConstants.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/TestConstants.java @@ -110,6 +110,7 @@ import org.eclipse.ditto.internal.utils.cluster.DistPubSubAccess; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.config.ScopedConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.config.PingConfig; import org.eclipse.ditto.internal.utils.protocol.config.ProtocolConfig; import org.eclipse.ditto.internal.utils.pubsub.StreamingType; @@ -126,9 +127,11 @@ import org.eclipse.ditto.protocol.TopicPath; import org.eclipse.ditto.protocol.adapter.DittoProtocolAdapter; import org.eclipse.ditto.things.model.Attributes; +import org.eclipse.ditto.things.model.FeatureProperties; import org.eclipse.ditto.things.model.Thing; import org.eclipse.ditto.things.model.ThingId; import org.eclipse.ditto.things.model.signals.commands.modify.ModifyThing; +import org.eclipse.ditto.things.model.signals.events.FeatureDesiredPropertiesModified; import org.eclipse.ditto.things.model.signals.events.ThingModified; import org.eclipse.ditto.things.model.signals.events.ThingModifiedEvent; import org.mockito.Mockito; @@ -325,6 +328,14 @@ public static final class Things { } + public static final class Feature { + + public static final String FEATURE_ID = "Feature"; + public static final FeatureProperties FEATURE_DESIRED_PROPERTIES = FeatureProperties.newBuilder() + .set("property", "test").build(); + + } + public static final class Authorization { public static final String SUBJECT_ID = "some:subject"; @@ -924,7 +935,8 @@ public static ActorRef createConnectionSupervisorActor(final ConnectionId connec final var enforcerActorPropsFactory = ConnectionEnforcerActorPropsFactory.get(actorSystem, dittoExtensionsConfig); final Props props = - ConnectionSupervisorActor.props(commandForwarderActor, pubSubMediator, enforcerActorPropsFactory); + ConnectionSupervisorActor.props(commandForwarderActor, pubSubMediator, enforcerActorPropsFactory, + Mockito.mock(MongoReadJournal.class)); final Props shardRegionMockProps = Props.create(ShardRegionMockActor.class, props, connectionId.toString()); @@ -984,6 +996,12 @@ public static ThingModifiedEvent thingModified(final Collection featureDesiredPropertiesModified(Collection readSubjects) { + DittoHeaders dittoHeaders = DittoHeaders.newBuilder().readGrantedSubjects(readSubjects).build(); + return FeatureDesiredPropertiesModified.of(Things.THING_ID, Feature.FEATURE_ID, + Feature.FEATURE_DESIRED_PROPERTIES, 1, null, dittoHeaders, null); + } + public static MessageCommand sendThingMessage(final Collection readSubjects) { final DittoHeaders dittoHeaders = DittoHeaders.newBuilder() .readGrantedSubjects(readSubjects) diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java index 6a72dd50171..d2b7344b440 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java @@ -19,6 +19,8 @@ import java.io.IOException; import java.io.InputStreamReader; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -28,6 +30,7 @@ import org.assertj.core.api.Assertions; import org.eclipse.ditto.connectivity.model.Connection; +import org.eclipse.ditto.connectivity.model.ConnectionId; import org.eclipse.ditto.connectivity.model.ConnectivityModelFactory; import org.eclipse.ditto.connectivity.model.HonoAddressAlias; import org.eclipse.ditto.connectivity.model.ReplyTarget; @@ -118,10 +121,11 @@ private Connection getExpectedHonoConnection(final Connection originalConnection "correlation-id", "{{ header:correlation-id }}", "subject", "{{ header:subject | fn:default(topic:action-subject) }}" ); + final var connectionId = originalConnection.getId(); return ConnectivityModelFactory.newConnectionBuilder(originalConnection) .uri(honoConfig.getBaseUri().toString().replaceFirst("(\\S+://)(\\S+)", - "$1" + honoConfig.getUserPasswordCredentials().getUsername() - + ":" + honoConfig.getUserPasswordCredentials().getPassword() + "$1" + URLEncoder.encode(honoConfig.getUserPasswordCredentials().getUsername(), StandardCharsets.UTF_8) + + ":" + URLEncoder.encode(honoConfig.getUserPasswordCredentials().getPassword(), StandardCharsets.UTF_8) + "@$2")) .validateCertificate(honoConfig.isValidateCertificates()) .specificConfig(Map.of( @@ -131,22 +135,22 @@ private Connection getExpectedHonoConnection(final Connection originalConnection ) .setSources(List.of( ConnectivityModelFactory.newSourceBuilder(sourcesByAddress.get(TELEMETRY.getAliasValue())) - .addresses(Set.of(getExpectedResolvedSourceAddress(TELEMETRY))) + .addresses(Set.of(getExpectedResolvedSourceAddress(TELEMETRY, connectionId))) .replyTarget(ReplyTarget.newBuilder() - .address(getExpectedResolvedCommandTargetAddress()) + .address(getExpectedResolvedCommandTargetAddress(connectionId)) .headerMapping(commandReplyTargetHeaderMapping) .build()) .build(), ConnectivityModelFactory.newSourceBuilder(sourcesByAddress.get(EVENT.getAliasValue())) - .addresses(Set.of(getExpectedResolvedSourceAddress(EVENT))) + .addresses(Set.of(getExpectedResolvedSourceAddress(EVENT, connectionId))) .replyTarget(ReplyTarget.newBuilder() - .address(getExpectedResolvedCommandTargetAddress()) + .address(getExpectedResolvedCommandTargetAddress(connectionId)) .headerMapping(commandReplyTargetHeaderMapping) .build()) .build(), ConnectivityModelFactory.newSourceBuilder( sourcesByAddress.get(COMMAND_RESPONSE.getAliasValue())) - .addresses(Set.of(getExpectedResolvedSourceAddress(COMMAND_RESPONSE))) + .addresses(Set.of(getExpectedResolvedSourceAddress(COMMAND_RESPONSE, connectionId))) .headerMapping(ConnectivityModelFactory.newHeaderMapping(Map.of( "correlation-id", "{{ header:correlation-id }}", "status", "{{ header:status }}" @@ -155,8 +159,8 @@ private Connection getExpectedHonoConnection(final Connection originalConnection )) .setTargets(List.of( ConnectivityModelFactory.newTargetBuilder(targets.get(0)) - .address(getExpectedResolvedCommandTargetAddress()) - .originalAddress(getExpectedResolvedCommandTargetAddress()) + .address(getExpectedResolvedCommandTargetAddress(connectionId)) + .originalAddress(getExpectedResolvedCommandTargetAddress(connectionId)) .headerMapping(ConnectivityModelFactory.newHeaderMapping( Stream.concat( basicAdditionalTargetHeaderMappingEntries.entrySet().stream(), @@ -166,8 +170,8 @@ private Connection getExpectedHonoConnection(final Connection originalConnection )) .build(), ConnectivityModelFactory.newTargetBuilder(targets.get(1)) - .address(getExpectedResolvedCommandTargetAddress()) - .originalAddress(getExpectedResolvedCommandTargetAddress()) + .address(getExpectedResolvedCommandTargetAddress(connectionId)) + .originalAddress(getExpectedResolvedCommandTargetAddress(connectionId)) .headerMapping(ConnectivityModelFactory.newHeaderMapping( basicAdditionalTargetHeaderMappingEntries )) @@ -182,12 +186,12 @@ private static Map getSourcesByAddress(final Iterable so return result; } - private static String getExpectedResolvedSourceAddress(final HonoAddressAlias honoAddressAlias) { - return "hono." + honoAddressAlias.getAliasValue(); + private static String getExpectedResolvedSourceAddress(final HonoAddressAlias honoAddressAlias, final ConnectionId connectionId) { + return "hono." + honoAddressAlias.getAliasValue() + "." + connectionId; } - private static String getExpectedResolvedCommandTargetAddress() { - return "hono." + HonoAddressAlias.COMMAND.getAliasValue() + "/{{thing:id}}"; + private static String getExpectedResolvedCommandTargetAddress(final ConnectionId connectionId) { + return "hono." + HonoAddressAlias.COMMAND.getAliasValue() + "." + connectionId + "/{{thing:id}}"; } } diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformerTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformerTest.java index 7b10149aef7..b6c4ff97a77 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformerTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformerTest.java @@ -276,4 +276,32 @@ public void transformExternalMessageWithInvalidRetainValueYieldsTransformationFa .hasCauseInstanceOf(InvalidHeaderValueException.class); } + @Test + public void transformFullyFledgedExternalMessageWithBlankHeaderReturnsExpectedTransformationSuccessResult() { + final var correlationId = testNameCorrelationId.getCorrelationId(); + final var genericMqttPublish = GenericMqttPublish.builder(MQTT_TOPIC, MQTT_QOS) + .retain(RETAIN) + .payload(PAYLOAD) + .correlationData(ByteBufferUtils.fromUtf8String(correlationId.toString())) + .contentType(CONTENT_TYPE.getValue()) + .responseTopic(REPLY_TO_TOPIC) + .userProperties(USER_PROPERTIES) + .build(); + Mockito.when(externalMessage.getHeaders()) + .thenReturn(DittoHeaders.newBuilder() + .putHeader(MqttHeader.MQTT_TOPIC.getName(), MQTT_TOPIC.toString()) + .putHeader(MqttHeader.MQTT_QOS.getName(), String.valueOf(MQTT_QOS.getCode())) + .putHeader(MqttHeader.MQTT_RETAIN.getName(), String.valueOf(genericMqttPublish.isRetain())) + .putHeader("ablankheader", "") + .correlationId(correlationId) + .putHeader(ExternalMessage.REPLY_TO_HEADER, REPLY_TO_TOPIC.toString()) + .contentType(CONTENT_TYPE) + .putHeaders(USER_PROPERTIES.stream() + .collect(Collectors.toMap(UserProperty::name, UserProperty::value))) + .build()); + Mockito.when(externalMessage.getBytePayload()).thenReturn(genericMqttPublish.getPayload()); + + assertThat(ExternalMessageToMqttPublishTransformer.transform(externalMessage, mqttPublishTarget)) + .isEqualTo(TransformationSuccess.of(externalMessage, genericMqttPublish)); + } } \ No newline at end of file diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java index e268f81ff06..5fc675a4de2 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.io.InputStreamReader; -import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Collections; @@ -28,12 +27,16 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + +import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistenceResponse; import org.eclipse.ditto.base.model.correlationid.TestNameCorrelationId; import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.api.BaseClientState; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionConfigurationInvalidException; @@ -82,15 +85,19 @@ import org.eclipse.ditto.internal.utils.akka.ActorSystemResource; import org.eclipse.ditto.internal.utils.akka.PingCommand; import org.eclipse.ditto.internal.utils.akka.controlflow.WithSender; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import org.eclipse.ditto.internal.utils.test.Retry; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.thingsearch.model.signals.commands.subscription.CreateSubscription; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; +import org.mockito.Mockito; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigValueFactory; @@ -210,8 +217,14 @@ public void testConnection() { @Test public void testConnectionTypeHono() throws IOException { //GIVEN - final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json"); - final var expectedHonoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-expected.json"); + final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json", null) + .toBuilder() + .id(connectionId) + .build(); + final var expectedHonoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-expected.json", connectionId) + .toBuilder() + .id(connectionId) + .build(); final var testConnection = TestConnection.of(honoConnection, dittoHeadersWithCorrelationId); final var testProbe = actorSystemResource1.newTestProbe(); final var connectionSupervisorActor = createSupervisor(); @@ -234,7 +247,7 @@ public void testConnectionTypeHono() throws IOException { @Test public void testRestartByConnectionType() throws IOException { // GIVEN - final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json"); + final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json", null); mockClientActorProbe.setAutoPilot(new TestActor.AutoPilot() { @Override public TestActor.AutoPilot run(final ActorRef sender, final Object msg) { @@ -248,13 +261,17 @@ public TestActor.AutoPilot run(final ActorRef sender, final Object msg) { final var connectionActorProps = Props.create(ConnectionPersistenceActor.class, () -> new ConnectionPersistenceActor(connectionId, + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), Trilean.TRUE, ConfigFactory.empty())); final var underTest = actorSystemResource1.newActor(connectionActorProps, connectionId.toString()); - underTest.tell(createConnection(honoConnection), testProbe.ref()); + final CreateConnection createConnection = createConnection( + honoConnection.toBuilder().id(connectionId).build() + ); + underTest.tell(createConnection, testProbe.ref()); testProbe.expectMsgClass(FiniteDuration.apply(20, "s"), CreateConnectionResponse.class); Arrays.stream(ConnectionType.values()).forEach(connectionType -> { @@ -270,13 +287,19 @@ public TestActor.AutoPilot run(final ActorRef sender, final Object msg) { }); } - private static Connection generateConnectionObjectFromJsonFile(final String fileName) throws IOException { + private static Connection generateConnectionObjectFromJsonFile(final String fileName, + @Nullable ConnectionId connectionId) throws IOException { final var testClassLoader = DefaultHonoConnectionFactoryTest.class.getClassLoader(); try (final var connectionJsonFileStreamReader = new InputStreamReader( testClassLoader.getResourceAsStream(fileName) )) { - return ConnectivityModelFactory.connectionFromJson( - JsonFactory.readFrom(connectionJsonFileStreamReader).asObject()); + JsonObject jsonObject = JsonFactory.readFrom(connectionJsonFileStreamReader).asObject(); + var connId = jsonObject.getValue("id"); + if (connectionId != null && connId.isPresent()) { + var jsonString = jsonObject.formatAsString().replace(connId.get().asString(), connectionId); + jsonObject = JsonFactory.readFrom(jsonString).asObject(); + } + return ConnectivityModelFactory.connectionFromJson(jsonObject); } } @@ -350,7 +373,7 @@ public void manageConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // close connection final CloseConnection closeConnection = CloseConnection.of(connectionId, dittoHeadersWithCorrelationId); @@ -375,7 +398,7 @@ public void deleteConnectionUpdatesSubscriptionsAndClosesConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // delete connection underTest.tell(DeleteConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -465,7 +488,7 @@ public void createConnectionAfterDeleted() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // delete connection underTest.tell(DeleteConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -476,7 +499,7 @@ public void createConnectionAfterDeleted() { // create connection again (while ConnectionActor is in deleted state) underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); } @Test @@ -488,7 +511,7 @@ public void openConnectionAfterDeletedFails() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // delete connection underTest.tell(DeleteConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -509,7 +532,7 @@ public void createConnectionInClosedState() { // create connection underTest.tell(createConnection(closedConnection), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse(closedConnection)); + expectCreateConnectionResponse(testProbe, closedConnection); // assert that client actor is not called for closed connection mockClientActorProbe.expectNoMessage(); @@ -674,7 +697,7 @@ public void modifyConnectionInClosedState() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // close connection underTest.tell(CloseConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -697,7 +720,7 @@ public void retrieveMetricsInClosedStateDoesNotStartClientActor() { // create connection underTest.tell(createConnection(closedConnection), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse(closedConnection)); + expectCreateConnectionResponse(testProbe, closedConnection); mockClientActorProbe.expectNoMessage(); // retrieve metrics @@ -721,7 +744,7 @@ public void modifyConnectionClosesAndRestartsClientActor() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // modify connection | Implicitly validates the restart by waiting for pubsub subscribe from client actor. underTest.tell(ModifyConnection.of(connection, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -746,7 +769,7 @@ public void recoverOpenConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // stop actor actorSystemResource1.stopActor(underTest); @@ -797,7 +820,7 @@ public void recoverModifiedConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // modify connection final var modifiedConnection = ConnectivityModelFactory.newConnectionBuilder(connection) @@ -819,7 +842,11 @@ public void recoverModifiedConnection() { // retrieve connection status underTest.tell(RetrieveConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); - testProbe.expectMsg(RetrieveConnectionResponse.of(modifiedConnection.toJson(), dittoHeadersWithCorrelationId)); + testProbe.expectMsg(RetrieveConnectionResponse.of(modifiedConnection.toJson(), + dittoHeadersWithCorrelationId.toBuilder() + .eTag(EntityTag.fromString("\"rev:2\"")) + .build() + )); } @Test @@ -831,7 +858,7 @@ public void recoverClosedConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // close connection underTest.tell(CloseConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -873,7 +900,7 @@ public void recoverDeletedConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // delete connection underTest.tell(DeleteConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -893,14 +920,16 @@ public void recoverDeletedConnection() { @Test public void exceptionDuringClientActorPropsCreation() { - final var connectionActorProps = ConnectionPersistenceActor.props( - TestConstants.createRandomConnectionId(), commandForwarderActor, pubSubMediatorProbe.ref(), - ConfigFactory.empty() - ); - final var testProbe = actorSystemResource1.newTestProbe(); final var supervisor = actorSystemResource1.newTestProbe(); - final var connectionActorRef = supervisor.childActorOf(connectionActorProps); + + final var connectionActorProps = ConnectionPersistenceActor.props( + connectionId, + Mockito.mock(MongoReadJournal.class), + commandForwarderActor, + pubSubMediatorProbe.ref(), + ConfigFactory.empty()); + final var connectionActorRef = supervisor.childActorOf(connectionActorProps, "connection"); // create connection final CreateConnection createConnection = createConnection(); @@ -920,7 +949,8 @@ public void exceptionDuringClientActorPropsCreation() { @Test public void exceptionDueToCustomValidator() { - final var connectionActorProps = ConnectionPersistenceActor.props(TestConstants.createRandomConnectionId(), + final var connectionActorProps = ConnectionPersistenceActor.props(connectionId, + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), ConfigFactory.empty()); @@ -954,7 +984,7 @@ public void testResetConnectionMetrics() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // reset metrics final var resetConnectionMetrics = ResetConnectionMetrics.of(connectionId, dittoHeadersWithCorrelationId); @@ -973,7 +1003,7 @@ public void testConnectionActorRespondsToCleanupCommand() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // send cleanup command underTest.tell(CleanupPersistence.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -989,7 +1019,7 @@ public void enableConnectionLogs() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); //Close logging which are automatically enabled via create connection underTest.tell(LoggingExpired.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -1011,7 +1041,7 @@ public void retrieveLogsInClosedStateDoesNotStartClientActor() { // create connection underTest.tell(createConnection(closedConnection), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse(closedConnection)); + expectCreateConnectionResponse(testProbe, closedConnection); clientActorProbe.expectNoMessage(); // retrieve logs @@ -1042,7 +1072,7 @@ public void retrieveLogsIsAggregated() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // retrieve logs final var retrieveConnectionLogs = RetrieveConnectionLogs.of(connectionId, dittoHeadersWithCorrelationId); @@ -1069,7 +1099,7 @@ public void resetConnectionLogs() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // reset logs underTest.tell(resetConnectionLogs, testProbe.ref()); @@ -1086,7 +1116,7 @@ public void enabledConnectionLogsAreEnabledAgainAfterModify() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // Wait until connection is established // enable connection logs @@ -1115,7 +1145,7 @@ public void disabledConnectionLogsAreNotEnabledAfterModify() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); //Close logging which are automatically enabled via create connection underTest.tell(LoggingExpired.of(connectionId, DittoHeaders.empty()), testProbe.ref()); @@ -1155,6 +1185,7 @@ public TestActor.AutoPilot run(final ActorRef sender, final Object msg) { final var testProbe = actorSystemResource1.newTestProbe(); final var connectionActorProps = Props.create(ConnectionPersistenceActor.class, () -> new ConnectionPersistenceActor(connectionId, + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), Trilean.TRUE, @@ -1211,12 +1242,14 @@ private static boolean isMessageSenderInstanceOf(final Object message, final Cla } @Test + @Ignore("TODO unignore and stabilize flaky test") public void retriesStartingClientActor() { final var parent = actorSystemResource1.newTestProbe(); final var underTest = parent.childActorOf( Props.create( ConnectionPersistenceActor.class, () -> new ConnectionPersistenceActor(connectionId, + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), Trilean.FALSE, @@ -1230,9 +1263,13 @@ public void retriesStartingClientActor() { final DittoHeaders headersIndicatingFailingInstantiation = createConnection.getDittoHeaders().toBuilder() .putHeader("number-of-instantiation-failures", String.valueOf(TestConstants.CONNECTION_CONFIG.getClientActorRestartsBeforeEscalation())) + .responseRequired(false) .build(); underTest.tell(createConnection.setDittoHeaders(headersIndicatingFailingInstantiation), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse().setDittoHeaders(headersIndicatingFailingInstantiation)); + final CreateConnectionResponse resp = + expectCreateConnectionResponse(testProbe, connection); + assertThat(resp.getDittoHeaders()) + .isEqualTo(headersIndicatingFailingInstantiation); assertThat(underTest.isTerminated()).isFalse(); } @@ -1244,6 +1281,7 @@ public void escalatesWhenClientActorFailsTooOften() { Props.create( ConnectionPersistenceActor.class, () -> new ConnectionPersistenceActor(connectionId, + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), Trilean.FALSE, @@ -1257,9 +1295,13 @@ public void escalatesWhenClientActorFailsTooOften() { final DittoHeaders headersIndicatingFailingInstantiation = createConnection.getDittoHeaders().toBuilder() .putHeader("number-of-instantiation-failures", String.valueOf(TestConstants.CONNECTION_CONFIG.getClientActorRestartsBeforeEscalation() + 1)) + .responseRequired(false) .build(); underTest.tell(createConnection.setDittoHeaders(headersIndicatingFailingInstantiation), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse().setDittoHeaders(headersIndicatingFailingInstantiation)); + final CreateConnectionResponse resp = + expectCreateConnectionResponse(testProbe, connection); + assertThat(resp.getDittoHeaders()) + .isEqualTo(headersIndicatingFailingInstantiation); testProbe.expectTerminated(underTest, FiniteDuration.apply(3, TimeUnit.SECONDS)); } @@ -1272,7 +1314,7 @@ public void deleteConnectionCommandEmitsEvent() { // create connection underTest.tell(createConnection(closedConnection), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse(closedConnection)); + expectCreateConnectionResponse(testProbe, closedConnection); pubSubMediatorProbe.expectMsgClass(DistributedPubSubMediator.Subscribe.class); // delete connection @@ -1301,14 +1343,17 @@ private CreateConnection createConnection(final Connection connection) { return CreateConnection.of(connection, dittoHeadersWithCorrelationId); } - private CreateConnectionResponse createConnectionResponse() { - return createConnectionResponse(connection); - } - - private CreateConnectionResponse createConnectionResponse(final Connection connection) { - return CreateConnectionResponse.of(connection, dittoHeadersWithCorrelationId); + private CreateConnectionResponse expectCreateConnectionResponse(final TestProbe probe, + final Connection theConnection) { + final CreateConnectionResponse resp = + probe.expectMsgClass(CreateConnectionResponse.class); + Assertions.assertThat(resp.getConnection()) + .usingRecursiveComparison() + .ignoringFields("revision", "modified", "created") + .isEqualTo(theConnection); + return resp; } - + private void simulateSuccessfulOpenConnectionInClientActor() { expectMockClientActorMessage(EnableConnectionLogs.of(connectionId, DittoHeaders.empty())); expectMockClientActorMessage(OpenConnection.of(connectionId, dittoHeadersWithCorrelationId)); diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceOperationsActorIT.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceOperationsActorIT.java index 883749f2701..af6b7f9171e 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceOperationsActorIT.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceOperationsActorIT.java @@ -34,10 +34,12 @@ import org.eclipse.ditto.connectivity.service.enforcement.ConnectionEnforcerActorPropsFactory; import org.eclipse.ditto.internal.utils.config.ScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.ops.eventsource.MongoEventSourceITAssertions; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.utils.jsr305.annotations.AllValuesAreNonnullByDefault; import org.junit.ClassRule; import org.junit.Test; +import org.mockito.Mockito; import com.typesafe.config.Config; @@ -134,7 +136,8 @@ protected ActorRef startEntityActor(final ActorSystem system, final ActorRef pub final var dittoExtensionsConfig = ScopedConfig.dittoExtension(system.settings().config()); final var enforcerActorPropsFactory = ConnectionEnforcerActorPropsFactory.get(system, dittoExtensionsConfig); final Props props = - ConnectionSupervisorActor.props(proxyActorProbe.ref(), pubSubMediator, enforcerActorPropsFactory); + ConnectionSupervisorActor.props(proxyActorProbe.ref(), pubSubMediator, enforcerActorPropsFactory, + Mockito.mock(MongoReadJournal.class)); return system.actorOf(props, String.valueOf(id)); } diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/SignalFilterWithFilterTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/SignalFilterWithFilterTest.java index dee549fae0d..41cbdbac662 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/SignalFilterWithFilterTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/SignalFilterWithFilterTest.java @@ -23,6 +23,8 @@ import java.util.Collections; import java.util.List; +import org.assertj.core.api.Assertions; +import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.connectivity.service.messaging.monitoring.ConnectionMonitorRegistry; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.json.JsonValue; @@ -253,4 +255,21 @@ public void applySignalFilterWithNamespacesAndRqlFilter() { targetD); // THEN: only targetA and targetD should be in the filtered targets } + /** + * Test that target filtering works also for desired properties events. Issue #1599 + */ + @Test + public void applySignalFilterOnFeatureDesiredPropertiesModified() { + Target target = ConnectivityModelFactory.newTargetBuilder().address("address") + .authorizationContext(newAuthContext(DittoAuthorizationContextType.UNSPECIFIED, AUTHORIZED)) + .topics(ConnectivityModelFactory.newFilteredTopicBuilder(TWIN_EVENTS) + .withFilter("like(resource:path,'/features/" + TestConstants.Feature.FEATURE_ID + "*')") + .build()).build(); + Connection connection = TestConstants.createConnection(CONNECTION_ID, target); + SignalFilter signalFilter = new SignalFilter(connection, connectionMonitorRegistry); + Signal signal = TestConstants.featureDesiredPropertiesModified(Collections.singletonList(AUTHORIZED)); + + List filteredTargets = signalFilter.filter(signal); + Assertions.assertThat(filteredTargets).hasSize(1).contains(target); + } } diff --git a/connectivity/service/src/test/resources/hono-connection-custom-expected.json b/connectivity/service/src/test/resources/hono-connection-custom-expected.json index c55d5c52168..6ccab3c12a8 100644 --- a/connectivity/service/src/test/resources/hono-connection-custom-expected.json +++ b/connectivity/service/src/test/resources/hono-connection-custom-expected.json @@ -3,11 +3,11 @@ "name": "Things-Hono Test 1", "connectionType": "hono", "connectionStatus": "open", - "uri": "tcp://test_username:test_password@localhost:9922", + "uri": "tcp://test_username:test_password_w%2Fspecial_char@localhost:9922", "sources": [ { "addresses": [ - "hono.telemetry" + "hono.telemetry.test-connection-id" ], "consumerCount": 1, "qos": 0, @@ -32,7 +32,7 @@ "implicitStandaloneThingCreation" ], "replyTarget": { - "address": "hono.command/{{thing:id}}", + "address": "hono.command.test-connection-id/{{thing:id}}", "headerMapping": { "device_id": "custom_value1", "user_key1": "user_value1", @@ -48,7 +48,7 @@ }, { "addresses": [ - "hono.event" + "hono.event.test-connection-id" ], "consumerCount": 1, "qos": 1, @@ -72,7 +72,7 @@ "implicitStandaloneThingCreation" ], "replyTarget": { - "address": "hono.command/{{thing:id}}", + "address": "hono.command.test-connection-id/{{thing:id}}", "headerMapping": { "device_id": "{{ thing:id }}", "subject": "custom_value2", @@ -88,7 +88,7 @@ }, { "addresses": [ - "hono.command_response" + "hono.command_response.test-connection-id" ], "consumerCount": 1, "qos": 0, @@ -120,7 +120,7 @@ ], "targets": [ { - "address": "hono.command/{{thing:id}}", + "address": "hono.command.test-connection-id/{{thing:id}}", "topics": [ "_/_/things/live/messages", "_/_/things/live/commands" @@ -137,7 +137,7 @@ } }, { - "address": "hono.command/{{thing:id}}", + "address": "hono.command.test-connection-id/{{thing:id}}", "topics": [ "_/_/things/twin/events", "_/_/things/live/events" diff --git a/connectivity/service/src/test/resources/test.conf b/connectivity/service/src/test/resources/test.conf index b4ec24e9b1c..a4676ce04ff 100644 --- a/connectivity/service/src/test/resources/test.conf +++ b/connectivity/service/src/test/resources/test.conf @@ -103,7 +103,7 @@ ditto { sasl-mechanism = PLAIN bootstrap-servers = "tcp://server1:port1,tcp://server2:port2,tcp://server3:port3" username = test_username - password = test_password + password = test_password_w/special_char } connection { diff --git a/deployment/README.md b/deployment/README.md index 92fdf572fdf..944fa99dc7a 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -2,11 +2,11 @@ In order to deploy/start Ditto, you have the following options: +- [starting via Kubernetes and Helm](helm/README.md) (* preferred way for productive setup) - [starting via Docker and docker-compose](docker/README.md) - [starting via Kubernetes with k3s](kubernetes/k3s/README.md) - [starting via Kubernetes with minikube](kubernetes/minikube/README.md) - [starting via OpenShift](openshift/README.md) -- [starting via Kubernetes and Helm](helm/README.md) - [starting via Microsoft Azure](azure/README.md) ### Resource requirements diff --git a/deployment/docker/README.md b/deployment/docker/README.md index 87631e977a9..ffc4874e221 100755 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -49,7 +49,7 @@ curl http://devops:foobar@localhost:8080/devops/config/gateway/?path=ditto ``` Or by going through the configuration files in this repository, all available configuration files are -[linked here](https://www.eclipse.org/ditto/installation-operating.html#ditto-configuration). +[linked here](https://www.eclipse.dev/ditto/installation-operating.html#ditto-configuration). ## Start Eclipse Ditto diff --git a/deployment/docker/nginx.conf b/deployment/docker/nginx.conf index 54586d8f8b1..1efa8b9a1f6 100755 --- a/deployment/docker/nginx.conf +++ b/deployment/docker/nginx.conf @@ -1,6 +1,8 @@ -worker_processes 1; +worker_processes auto; -events {worker_connections 1024;} +events { + worker_connections 1024; +} http { include /etc/nginx/mime.types; diff --git a/deployment/docker/sandbox/docker-compose.yml b/deployment/docker/sandbox/docker-compose.yml index aa070a754c1..8a8a2320a07 100644 --- a/deployment/docker/sandbox/docker-compose.yml +++ b/deployment/docker/sandbox/docker-compose.yml @@ -135,7 +135,7 @@ services: - TZ=Europe/Berlin - BIND_HOSTNAME=0.0.0.0 - ENABLE_PRE_AUTHENTICATION=true - - DEVOPS_SECURE_STATUS=false + - DEVOPS_STATUS_SECURED=false - DITTO_DEVOPS_FEATURE_WOT_INTEGRATION_ENABLED=true - | JAVA_TOOL_OPTIONS= diff --git a/deployment/docker/sandbox/html/index.html b/deployment/docker/sandbox/html/index.html index 4bcc6313894..cd904d960fe 100644 --- a/deployment/docker/sandbox/html/index.html +++ b/deployment/docker/sandbox/html/index.html @@ -105,7 +105,7 @@

- API.
The Swagger UI for exploring the API will require authentication.

- Visit the Eclipse Ditto documentation in + Visit the Eclipse Ditto documentation in order to learn more about the project.

diff --git a/deployment/docker/sandbox/nginx.conf b/deployment/docker/sandbox/nginx.conf index 66a854e8050..2cc9ce73cf8 100644 --- a/deployment/docker/sandbox/nginx.conf +++ b/deployment/docker/sandbox/nginx.conf @@ -1,6 +1,8 @@ -worker_processes 1; +worker_processes auto; -events {worker_connections 1024;} +events { + worker_connections 1024; +} http { charset utf-8; diff --git a/deployment/helm/README.md b/deployment/helm/README.md index 7a15958ac0f..85020b4a66a 100644 --- a/deployment/helm/README.md +++ b/deployment/helm/README.md @@ -1,21 +1,20 @@ # Eclipse Ditto :: Helm -The Ditto Helm chart is managed at the [Eclipse IoT Packages](https://github.com/eclipse/packages/tree/master/charts/ditto) project. +The official Ditto Helm chart is managed here, in folder [ditto](ditto/). +It is deployed as "OCI artifact" to Docker Hub at: https://hub.docker.com/r/eclipse/ditto ## Install Ditto via Helm Chart To install the chart with the release name eclipse-ditto, run the following commands: ```shell script -helm repo add eclipse-iot https://www.eclipse.org/packages/charts/ -helm repo update -helm install eclipse-ditto eclipse-iot/ditto +helm install -n ditto my-ditto oci://registry-1.docker.io/eclipse/ditto --version --wait ``` # Uninstall the Helm Chart -To uninstall/delete the eclipse-ditto deployment: +To uninstall/delete the `my-ditto` deployment: ```shell script -helm delete eclipse-ditto +helm uninstall my-ditto ``` diff --git a/deployment/helm/ditto/.gitignore b/deployment/helm/ditto/.gitignore new file mode 100644 index 00000000000..f791801bcae --- /dev/null +++ b/deployment/helm/ditto/.gitignore @@ -0,0 +1,2 @@ +charts/ +Chart.lock diff --git a/deployment/helm/ditto/.helmignore b/deployment/helm/ditto/.helmignore new file mode 100644 index 00000000000..d8882204716 --- /dev/null +++ b/deployment/helm/ditto/.helmignore @@ -0,0 +1,19 @@ +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deployment/helm/ditto/Chart.yaml b/deployment/helm/ditto/Chart.yaml new file mode 100644 index 00000000000..69ec7769588 --- /dev/null +++ b/deployment/helm/ditto/Chart.yaml @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +--- +apiVersion: v2 +name: ditto +description: | + Eclipse Ditto™ is a technology in the IoT implementing a software pattern called “digital twins”. + A digital twin is a virtual, cloud based, representation of his real world counterpart + (real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations etc). +type: application +version: 3.3.4 # chart version is effectively set by release-job +appVersion: 3.3.4 +keywords: + - iot-chart + - digital-twin + - IoT +home: https://www.eclipse.dev/ditto +sources: + - https://github.com/eclipse-ditto/ditto +icon: https://www.eclipse.dev/ditto/images/ditto.svg +maintainers: + - name: thjaeckle + email: thomas.jaeckle@beyonnex.io +dependencies: + - name: mongodb + repository: https://charts.bitnami.com/bitnami + version: ^12.x + condition: mongodb.enabled diff --git a/deployment/helm/ditto/README.md b/deployment/helm/ditto/README.md new file mode 100644 index 00000000000..631852cb0bb --- /dev/null +++ b/deployment/helm/ditto/README.md @@ -0,0 +1,137 @@ +# Eclipse Ditto + +## Introduction + +[Eclipse Ditto™](https://www.eclipse.dev/ditto/) is a technology in the IoT implementing a software pattern +called “digital twins”. A digital twin is a virtual, cloud based, representation of his real world counterpart +(real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations, …). + +This chart uses `eclipse/ditto-XXX` containers to run Ditto inside Kubernetes. + +## Prerequisites + +Installing Ditto using the chart requires the Helm tool to be installed as described on the +[IoT Packages chart repository prerequisites](https://www.eclipse.org/packages/prereqs/) page. + +TL;DR: + +* have a correctly configured [`kubectl`](https://kubernetes.io/docs/tasks/tools/#kubectl) (either against a local or remote k8s cluster) +* have [Helm installed](https://helm.sh/docs/intro/) + +The Helm chart is being tested to successfully install on the most recent Kubernetes versions. + +## Installing the Chart + +The instructions below illustrate how Ditto can be installed to the `ditto` namespace in a Kubernetes cluster using +release name `eclipse-ditto`. +The commands can easily be adapted to use a different namespace or release name. + +The target namespace in Kubernetes only needs to be created if it doesn't exist yet: + +```bash +kubectl create namespace ditto +``` + +The chart can then be installed to namespace `ditto` using release name `my-ditto`: + +```bash +helm install --dependency-update -n ditto my-ditto oci://registry-1.docker.io/eclipse/ditto --version --wait +``` + + +## Uninstalling the Chart + +To uninstall/delete the `my-ditto` release: + +```bash +helm uninstall -n ditto my-ditto +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +Please view the `values.yaml` for the list of possible configuration values with its documentation. + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example: + +```bash +helm install -n ditto my-ditto oci://registry-1.docker.io/eclipse/ditto --version --set swaggerui.enabled=false +``` + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. + +## Chart configuration options + +Please consult the [values.yaml](https://github.com/eclipse-ditto/ditto/blob/master/deployment/helm/ditto/values.yaml) +for all available configuration options of the Ditto Helm chart. + +### Scaling options + +Please note the defaults the chart comes with: +* the default deploy 1 instance per Ditto service +* each Ditto service is configured to require: + * 0.5 CPUs + * 1024 MiB of memory + +Adjust this to your requirements, e.g. scale horizontally by configuring a higher `replicaCount` or vertically by +configuring more resources. + +## Advanced configuration options + +Even more configuration options, not exposed to the `values.yaml`, can be configured using either environment variables +or Java "System properties". +To inspect all available configuration options, please inspect Ditto's service configurations: + +* [policies.conf](https://github.com/eclipse-ditto/ditto/blob/master/policies/service/src/main/resources/policies.conf) +* [things.conf](https://github.com/eclipse-ditto/ditto/blob/master/things/service/src/main/resources/things.conf) +* [things-search.conf](https://github.com/eclipse-ditto/ditto/blob/master/thingsearch/service/src/main/resources/search.conf) +* [connectivity.conf](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) +* [gateway.conf](https://github.com/eclipse-ditto/ditto/blob/master/gateway/service/src/main/resources/gateway.conf) + + +### Configuration via environment variables + +In order to provide an environment variable config overwrite, simply put the environment variable in the `extraEnv` +of the Ditto service you want to specify the configuration for. + +E.g. if you want to configure the `LOG_INCOMING_MESSAGES` for the `things` service to be disabled: +```yaml +things: + # ... + extraEnv: + - name: LOG_INCOMING_MESSAGES + value: "false" +``` + +### Configuration via system properties + +Not all Ditto/Akka configuration options have an environment variable overwrite defined in the configuration. +For configurations without such an environment variable overwrite, the option can be configured via Java system property. +The documentation on how this works can be found in the +[HOCON documentation](https://github.com/lightbend/config/blob/main/HOCON.md#conventional-override-by-system-properties), +which is the configuration format Ditto uses. + +E.g. if you want to adjust the `journal-collection` name which the `things` service uses to write its +journal entries to MongoDB (which can be found [here](https://github.com/eclipse-ditto/ditto/blob/33a38bc04b47d0167ba0e99fe76d96a54aa3d162/things/service/src/main/resources/things.conf#L268)), +simply configure: + + +```yaml +things: + # ... + systemProps: + - "-Dakka-contrib-mongodb-persistence-things-journal.overrides.journal-collection=another_fancy_name" +``` + + +## Troubleshooting + +If you experience high resource consumption (either CPU or RAM or both), you can limit the resource usage by +specifying resource limits. +This can be done individually for each single component. +Here is an example how to limit CPU to 0.25 Cores and RAM to 512 MiB for the `connectivity` service: + +```bash +helm upgrade -n ditto eclipse-ditto . --install --set connectivity.resources.limits.cpu=0.25 --set connectivity.resources.limits.memory=512Mi +``` diff --git a/deployment/helm/ditto/ci/ci-workflow-values.yaml b/deployment/helm/ditto/ci/ci-workflow-values.yaml new file mode 100644 index 00000000000..f47e2e3a19c --- /dev/null +++ b/deployment/helm/ditto/ci/ci-workflow-values.yaml @@ -0,0 +1,123 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +# +--- + +# profile for installing Ditto +# - with Ingress enabled and low cpu requests + +## ingress configuration +ingress: + enabled: true + host: ditto.example.com + annotations: + # kubernetes.io/tls-acme: "true" + tls: + - secretName: ditto-tls + hosts: + - ditto.example.com + +## ---------------------------------------------------------------------------- +## policies configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-policies.html +policies: + resources: + cpu: 0.15 + memoryMi: 512 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 50 + +## ---------------------------------------------------------------------------- +## things configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-things.html +things: + resources: + cpu: 0.15 + memoryMi: 512 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 50 + +## ---------------------------------------------------------------------------- +## things-search configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-things-search.html +thingsSearch: + resources: + cpu: 0.15 + memoryMi: 512 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 50 + +## ---------------------------------------------------------------------------- +## connectivity configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-connectivity.html +connectivity: + resources: + cpu: 0.15 + memoryMi: 768 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 55 + +## ---------------------------------------------------------------------------- +## gateway configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-gateway.html +gateway: + resources: + cpu: 0.15 + memoryMi: 512 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 50 + +## ---------------------------------------------------------------------------- +## nginx configuration +nginx: + resources: + cpu: 0.05 + memoryMi: 64 + initContainers: + waitForGateway: + enabled: false + +## ---------------------------------------------------------------------------- +## swaggerui configuration +swaggerui: + resources: + cpu: 0.05 + +## ---------------------------------------------------------------------------- +## dittoui configuration +dittoui: + resources: + cpu: 0.05 + +## ---------------------------------------------------------------------------- +## mongodb dependency chart configuration +mongodb: + enabled: true + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + readinessProbe: + enabled: false + livenessProbe: + enabled: false + auth: + enabled: false + persistence: + enabled: false diff --git a/deployment/helm/ditto/dittoui-config/nginx.conf b/deployment/helm/ditto/dittoui-config/nginx.conf new file mode 100644 index 00000000000..05e57d72a68 --- /dev/null +++ b/deployment/helm/ditto/dittoui-config/nginx.conf @@ -0,0 +1,36 @@ +worker_processes 1; + +error_log /var/log/nginx/error.log notice; +pid /tmp/nginx.pid; + +events { + worker_connections 1024; +} + +http { + proxy_temp_path /tmp/proxy_temp; + client_body_temp_path /tmp/client_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + server_tokens off; # Hide Nginx version + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/deployment/helm/ditto/local-values.yaml b/deployment/helm/ditto/local-values.yaml new file mode 100644 index 00000000000..faeb79351f3 --- /dev/null +++ b/deployment/helm/ditto/local-values.yaml @@ -0,0 +1,166 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +--- +# Default values for ditto. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + jwtOnly: false + basicAuthUsers: + ditto: + user: ditto + password: ditto + logging: + customConfigFile: + enabled: true + +## ---------------------------------------------------------------------------- +## policies configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-policies.html +policies: + resources: + cpu: 0.2 + memoryMi: 512 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 50 + config: + persistence: + events: + historicalHeadersToPersist: + - "ditto-originator" + - "ditto-origin" + - "correlation-id" + entityCreation: + grants: + - namespaces: + - "org.eclipse.ditto.room" + authSubjects: + - "connection:some" + +## ---------------------------------------------------------------------------- +## things configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-things.html +things: + resources: + cpu: 0.2 + memoryMi: 512 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 50 + config: + persistence: + events: + historicalHeadersToPersist: + - "ditto-originator" + - "ditto-origin" + - "correlation-id" + entityCreation: + grants: + - namespaces: + - "org.eclipse.ditto.room" + authSubjects: + - "connection:some" + +## ---------------------------------------------------------------------------- +## things-search configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-things-search.html +thingsSearch: + resources: + cpu: 0.2 + memoryMi: 512 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 50 + +## ---------------------------------------------------------------------------- +## connectivity configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-connectivity.html +connectivity: + resources: + cpu: 0.2 + memoryMi: 768 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 55 + +## ---------------------------------------------------------------------------- +## gateway configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-gateway.html +gateway: + resources: + cpu: 0.2 + memoryMi: 512 + jvm: + activeProcessorCount: 2 + heapRamPercentage: 50 + config: + authentication: + enablePreAuthentication: true + oauth: + openidConnectIssuers: + example: + issuer: "example.com" + authSubjects: + - "{{ jwt:sub }}" + - "{{ jwt:groups }}" + devops: + ## this controls whether /devops resource is secured or not + secured: true + authMethod: "basic" + oauth: + # configure the amount of clock skew in seconds to tolerate when verifying the local time against the exp and nbf claims + allowedClockSkew: 20s + openidConnectIssuers: + example-ops: + issuer: "example.com" + authSubjects: + - "{{ jwt:sub }}" + - "{{ jwt:groups }}" + oauthSubjects: + - "example-ops:devops-admin" + ## this controls whether /status resource is secured or not + statusSecured: true + statusAuthMethod: "basic" + # array of strings for subjects authorized to use "/status" API + statusOauthSubjects: + - "example-ops:devops-admin" + +## ---------------------------------------------------------------------------- +## nginx configuration +nginx: + resources: + cpu: 0.1 + memoryMi: 64 + initContainers: + waitForGateway: + enabled: false + +## ---------------------------------------------------------------------------- +## mongodb dependency chart configuration +mongodb: + enabled: false + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + readinessProbe: + enabled: false + livenessProbe: + enabled: false + auth: + enabled: false + persistence: + enabled: false diff --git a/deployment/helm/ditto/logback-config/connectivity.xml b/deployment/helm/ditto/logback-config/connectivity.xml new file mode 100644 index 00000000000..0d22766ad5c --- /dev/null +++ b/deployment/helm/ditto/logback-config/connectivity.xml @@ -0,0 +1,41 @@ + + + + + + sourceActorSystem + akkaUid + akkaTimestamp + x-correlation-id=correlation-id + connection-id=ditto-connection-id + connection-type=ditto-connection-type + + + + + + + + + + + + + + + + + + + diff --git a/deployment/helm/ditto/logback-config/gateway.xml b/deployment/helm/ditto/logback-config/gateway.xml new file mode 100755 index 00000000000..348e3a63875 --- /dev/null +++ b/deployment/helm/ditto/logback-config/gateway.xml @@ -0,0 +1,37 @@ + + + + + + sourceActorSystem + akkaUid + akkaTimestamp + x-correlation-id=correlation-id + + + + + + + + + + + + + + + + + diff --git a/deployment/helm/ditto/logback-config/policies.xml b/deployment/helm/ditto/logback-config/policies.xml new file mode 100755 index 00000000000..348e3a63875 --- /dev/null +++ b/deployment/helm/ditto/logback-config/policies.xml @@ -0,0 +1,37 @@ + + + + + + sourceActorSystem + akkaUid + akkaTimestamp + x-correlation-id=correlation-id + + + + + + + + + + + + + + + + + diff --git a/deployment/helm/ditto/logback-config/things.xml b/deployment/helm/ditto/logback-config/things.xml new file mode 100755 index 00000000000..348e3a63875 --- /dev/null +++ b/deployment/helm/ditto/logback-config/things.xml @@ -0,0 +1,37 @@ + + + + + + sourceActorSystem + akkaUid + akkaTimestamp + x-correlation-id=correlation-id + + + + + + + + + + + + + + + + + diff --git a/deployment/helm/ditto/logback-config/thingssearch.xml b/deployment/helm/ditto/logback-config/thingssearch.xml new file mode 100755 index 00000000000..348e3a63875 --- /dev/null +++ b/deployment/helm/ditto/logback-config/thingssearch.xml @@ -0,0 +1,37 @@ + + + + + + sourceActorSystem + akkaUid + akkaTimestamp + x-correlation-id=correlation-id + + + + + + + + + + + + + + + + + diff --git a/deployment/helm/ditto/nginx-config/ditto-down.svg b/deployment/helm/ditto/nginx-config/ditto-down.svg new file mode 100644 index 00000000000..679cf25b967 --- /dev/null +++ b/deployment/helm/ditto/nginx-config/ditto-down.svg @@ -0,0 +1,78 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + diff --git a/deployment/helm/ditto/nginx-config/ditto-up.svg b/deployment/helm/ditto/nginx-config/ditto-up.svg new file mode 100644 index 00000000000..33b074ba859 --- /dev/null +++ b/deployment/helm/ditto/nginx-config/ditto-up.svg @@ -0,0 +1,67 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + diff --git a/deployment/helm/ditto/nginx-config/index.html b/deployment/helm/ditto/nginx-config/index.html new file mode 100644 index 00000000000..abed686cd52 --- /dev/null +++ b/deployment/helm/ditto/nginx-config/index.html @@ -0,0 +1,174 @@ + + + + Eclipse Ditto + + + + + + + +

+ + + + + + diff --git a/deployment/helm/ditto/nginx-config/nginx-cors.conf b/deployment/helm/ditto/nginx-config/nginx-cors.conf new file mode 100644 index 00000000000..f9ed9330776 --- /dev/null +++ b/deployment/helm/ditto/nginx-config/nginx-cors.conf @@ -0,0 +1,39 @@ +# +# CORS header support +# +# As of Nginx 1.7.5, add_header supports an "always" parameter which +# allows CORS to work if the backend returns 4xx or 5xx status code. +# +# For more information on CORS, please see: http://enable-cors.org/ +# From this Gist: https://gist.github.com/Stanback/7145487 +# And this: https://gist.github.com/pauloricardomg/7084524 +# + +set $cors '1'; + +# OPTIONS indicates a CORS pre-flight request +if ($request_method = 'OPTIONS') { + set $cors "${cors}o"; +} + +if ($cors = '1') { + add_header 'Access-Control-Allow-Origin' '$http_origin' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, PATCH, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Headers' '$http_access_control_request_headers' always; + add_header 'Access-Control-Expose-Headers' '*' always; +} + +# OPTIONS (pre-flight) request from allowed CORS domain. return response directly +if ($cors = '1o') { + # Tell client that this pre-flight info is valid for 20 days + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Access-Control-Allow-Origin' '$http_origin' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, PATCH, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Headers' '$http_access_control_request_headers' always; + add_header 'Access-Control-Expose-Headers' '*' always; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 200; +} diff --git a/deployment/helm/ditto/swaggerui-config/index.html b/deployment/helm/ditto/swaggerui-config/index.html new file mode 100644 index 00000000000..460d0fe7bc6 --- /dev/null +++ b/deployment/helm/ditto/swaggerui-config/index.html @@ -0,0 +1,74 @@ + + + + + + Eclipse Ditto - HTTP API + + + + + + + + +
+ + + + + + + + diff --git a/deployment/helm/ditto/templates/NOTES.txt b/deployment/helm/ditto/templates/NOTES.txt new file mode 100644 index 00000000000..1b2fcbce4be --- /dev/null +++ b/deployment/helm/ditto/templates/NOTES.txt @@ -0,0 +1,33 @@ +Eclipse Ditto successfully installed! + +{{- if ( not .Values.openshift.routes.enabled ) }} +Access Ditto in your browser (http://localhost:8080) by running: + + kubectl --namespace {{ .Release.Namespace }} port-forward svc/{{ include "ditto.fullname" . }}-nginx 8080:8080 + +The /status resource can be accessed by: + + export STATUS_PWD=$(kubectl --namespace {{ .Release.Namespace }} get secret {{ include "ditto.fullname" . }}-gateway-secret -o jsonpath="{.data.status-password}" | base64 --decode) + curl -i -X GET "http://devops:${STATUS_PWD}@localhost:8080/status" + +The /devops resource can be accessed by: + + export DEVOPS_PWD=$(kubectl --namespace {{ .Release.Namespace }} get secret {{ include "ditto.fullname" . }}-gateway-secret -o jsonpath="{.data.devops-password}" | base64 --decode) + curl -i -X GET "http://devops:${DEVOPS_PWD}@localhost:8080/devops" +{{- else -}} +Access Ditto in your browser, get the URL with: + + echo https://$(kubectl --namespace {{ .Release.Namespace }} get route {{ include "ditto.fullname" . }} -o jsonpath="{.status.ingress[0].host}") + +The /status resource can be accessed by: + + export STATUS_PWD=$(kubectl --namespace {{ .Release.Namespace }} get secret {{ include "ditto.fullname" . }}-gateway-secret -o jsonpath="{.data.status-password}" | base64 --decode) + export URL=https://devops:${STATUS_PWD}@$(kubectl --namespace {{ .Release.Namespace }} get route {{ include "ditto.fullname" . }} -o jsonpath="{.status.ingress[0].host}") + curl -i -X GET "$URL/status" + +The /devops resource can be accessed by: + + export DEVOPS_PWD=$(kubectl --namespace {{ .Release.Namespace }} get secret {{ include "ditto.fullname" . }}-gateway-secret -o jsonpath="{.data.devops-password}" | base64 --decode) + export URL=https://devops:${DEVOPS_PWD}@$(kubectl --namespace {{ .Release.Namespace }} get route {{ include "ditto.fullname" . }} -o jsonpath="{.status.ingress[0].host}") + curl -i -X GET "$URL/devops" +{{- end }} diff --git a/deployment/helm/ditto/templates/_helpers.tpl b/deployment/helm/ditto/templates/_helpers.tpl new file mode 100644 index 00000000000..8ba1d57081f --- /dev/null +++ b/deployment/helm/ditto/templates/_helpers.tpl @@ -0,0 +1,72 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ditto.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ditto.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ditto.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "ditto.labels" -}} +helm.sh/chart: {{ include "ditto.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ditto.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "ditto.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified MongoDB server name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "ditto.mongodb.fullname" -}} +{{- if .Values.mongodb.fullnameOverride -}} +{{- .Values.mongodb.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "mongodb" .Values.mongodb.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/deployment/helm/ditto/templates/connectivity-deployment.yaml b/deployment/helm/ditto/templates/connectivity-deployment.yaml new file mode 100644 index 00000000000..b644969b478 --- /dev/null +++ b/deployment/helm/ditto/templates/connectivity-deployment.yaml @@ -0,0 +1,323 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.connectivity.enabled -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ditto.fullname" . }}-connectivity + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity +{{ include "ditto.labels" . | indent 4 }} +spec: + replicas: {{ .Values.connectivity.replicaCount }} + strategy: + {{- with .Values.connectivity.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.connectivity.minReadySeconds }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity + app.kubernetes.io/instance: {{ .Release.Name }} + actorSystemName: {{ .Values.akka.actorSystemName }} + {{- with .Values.connectivity.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.global.prometheus.enabled }} + prometheus.io/scrape: "true" + prometheus.io/path: "{{ .Values.global.prometheus.path }}" + prometheus.io/port: "{{ .Values.global.prometheus.port }}" + {{- end }} + checksum/mongodb-config: {{ include (print $.Template.BasePath "/mongodb-secret.yaml") . | sha256sum }} + {{- with .Values.connectivity.additionalAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.rbac.enabled }} + serviceAccountName: {{ template "ditto.serviceAccountName" . }} + {{- end }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + fsGroup: 1000 + initContainers: + {{- if .Values.global.logging.logFiles.enabled }} + - name: change-volume-owner + image: busybox + securityContext: + runAsUser: 0 + command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ] + volumeMounts: + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + containers: + - name: {{ .Chart.Name }}-connectivity + image: {{ printf "%s:%s" .Values.connectivity.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.connectivity.image.tag ) ) }} + imagePullPolicy: {{ .Values.connectivity.image.pullPolicy }} + env: + {{- if not .Values.global.logging.customConfigFile.enabled }} + - name: DITTO_LOGGING_DISABLE_SYSOUT_LOG + value: "{{ if .Values.global.logging.sysout.enabled }}false{{ else }}true{{ end }}" + - name: DITTO_LOGGING_FILE_APPENDER + value: "{{ if .Values.global.logging.logFiles.enabled }}true{{ else }}false{{ end }}" + {{- end }} + - name: DITTO_TRACING_ENABLED + value: "{{ .Values.global.tracing.enabled }}" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "{{ .Values.global.tracing.otelExporterOtlpEndpoint }}" + - name: DITTO_TRACING_SAMPLER + value: "{{ .Values.global.tracing.sampler }}" + - name: DITTO_TRACING_RANDOM_SAMPLER_PROBABILITY + value: "{{ .Values.global.tracing.randomSampler.probability }}" + - name: DITTO_TRACING_ADAPTIVE_SAMPLER_THROUGHPUT + value: "{{ .Values.global.tracing.adaptiveSampler.throughput }}" + {{- if .Values.global.logging.logstash.enabled }} + - name: DITTO_LOGGING_LOGSTASH_SERVER + value: "{{ .Values.global.logging.logstash.endpoint }}" + {{- end }} + - name: POD_LABEL_SELECTOR + value: "app.kubernetes.io/name=%s" + - name: POD_NAMESPACE + value: {{ .Release.Namespace }} + - name: INSTANCE_INDEX + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: DISCOVERY_METHOD + value: "kubernetes-api" + - name: TZ + value: "{{ .Values.global.timezone }}" + - name: JAVA_TOOL_OPTIONS + value: > + {{ .Values.global.jvmOptions }} + -XX:ActiveProcessorCount={{ .Values.connectivity.jvm.activeProcessorCount }} + -XX:MaxRAMPercentage={{ .Values.connectivity.jvm.heapRamPercentage }} + -XX:InitialRAMPercentage={{ .Values.connectivity.jvm.heapRamPercentage }} + -XX:MaxGCPauseMillis={{ .Values.connectivity.jvm.maxGcPauseMillis }} + {{ .Values.connectivity.additionalJvmOptions }} + {{- .Values.global.akkaOptions }} + {{- if .Values.global.logging.customConfigFile.enabled }} + -Dlogback.configurationFile=/opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{- range $index, $header := .Values.connectivity.config.persistence.events.historicalHeadersToPersist }} + "{{ printf "%s%d=%s" "-Dditto.connectivity.connection.event.historical-headers-to-persist." $index $header }}" + {{- end }} + {{ join " " .Values.connectivity.systemProps }} + - name: MONGO_DB_SSL_ENABLED + value: "{{ if .Values.dbconfig.connectivity.ssl }}true{{ else }}false{{ end }}" + - name: MONGO_DB_URI + valueFrom: + secretKeyRef: + name: {{ .Values.dbconfig.uriSecret | default ( printf "%s-mongodb-secret" ( include "ditto.fullname" . )) }} + key: connectivity-uri + - name: MONGO_DB_CONNECTION_MIN_POOL_SIZE + value: "{{ .Values.connectivity.config.mongodb.minPoolSize }}" + - name: MONGO_DB_CONNECTION_POOL_SIZE + value: "{{ .Values.connectivity.config.mongodb.maxPoolSize }}" + - name: MONGO_DB_CONNECTION_POOL_IDLE_TIME + value: "{{ .Values.connectivity.config.mongodb.maxPoolIdleTime }}" + - name: CLUSTER_BS_REQUIRED_CONTACTS + value: "{{ .Values.global.cluster.requiredContactPoints }}" + - name: DITTO_DDATA_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.ddata.numberOfShards }}" + - name: DITTO_DDATA_MAX_DELTA_ELEMENTS + value: "{{ .Values.global.cluster.ddata.maxDeltaElements }}" + - name: CLUSTER_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.numberOfShards }}" + - name: CLUSTER_DOWNING_STABLE_AFTER + value: "{{ .Values.global.cluster.downingStableAfter }}" + - name: CLUSTER_DOWNING_DOWN_ALL_WHEN_UNSTABLE + value: "{{ .Values.global.cluster.downAllWhenUnstable }}" + {{- if .Values.global.prometheus.enabled }} + - name: PROMETHEUS_PORT + value: "{{ .Values.global.prometheus.port }}" + {{- end }} + - name: AKKA_PERSISTENCE_MONGO_JOURNAL_WRITE_CONCERN + value: "{{ .Values.connectivity.config.mongodb.journalWriteConcern }}" + - name: AKKA_PERSISTENCE_MONGO_SNAPS_WRITE_CONCERN + value: "{{ .Values.connectivity.config.mongodb.snapsWriteConcern }}" + - name: BREAKER_MAXTRIES + value: "{{ .Values.connectivity.config.mongodb.journalCircuitBreaker.maxTries }}" + - name: BREAKER_TIMEOUT + value: "{{ .Values.connectivity.config.mongodb.journalCircuitBreaker.timeout }}" + - name: BREAKER_RESET + value: "{{ .Values.connectivity.config.mongodb.journalCircuitBreaker.reset }}" + - name: SNAPSHOT_BREAKER_MAXTRIES + value: "{{ .Values.connectivity.config.mongodb.snapsCircuitBreaker.maxTries }}" + - name: SNAPSHOT_BREAKER_TIMEOUT + value: "{{ .Values.connectivity.config.mongodb.snapsCircuitBreaker.timeout }}" + - name: SNAPSHOT_BREAKER_RESET + value: "{{ .Values.connectivity.config.mongodb.snapsCircuitBreaker.reset }}" + - name: CONNECTION_ACTIVITY_CHECK_INTERVAL + value: "{{ .Values.connectivity.config.persistence.activityCheckInterval }}" + - name: HEALTH_CHECK_METRICS_REPORTER_RESOLUTION + value: "{{ .Values.connectivity.config.cleanup.metricsReporter.resolution }}" + - name: HEALTH_CHECK_METRICS_REPORTER_HISTORY + value: "{{ .Values.connectivity.config.cleanup.metricsReporter.history }}" + - name: CLEANUP_ENABLED + value: "{{ .Values.connectivity.config.cleanup.enabled }}" + - name: CLEANUP_QUIET_PERIOD + value: "{{ .Values.connectivity.config.cleanup.quietPeriod }}" + - name: CLEANUP_HISTORY_RETENTION_DURATION + value: "{{ .Values.connectivity.config.cleanup.history.retentionDuration }}" + - name: CLEANUP_INTERVAL + value: "{{ .Values.connectivity.config.cleanup.interval }}" + - name: CLEANUP_TIMER_THRESHOLD + value: "{{ .Values.connectivity.config.cleanup.timerThreshold }}" + - name: CLEANUP_CREDITS_PER_BATCH + value: "{{ .Values.connectivity.config.cleanup.creditsPerBatch }}" + - name: CONNECTION_SNAPSHOT_INTERVAL + value: "{{ .Values.connectivity.config.persistence.snapshots.interval }}" + - name: CONNECTION_SNAPSHOT_THRESHOLD + value: "{{ .Values.connectivity.config.persistence.snapshots.threshold }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_ENABLED + value: "{{ .Values.connectivity.config.policiesEnforcer.cache.enabled }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_MAX_SIZE + value: "{{ .Values.connectivity.config.policiesEnforcer.cache.maxSize }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_EXPIRE_AFTER_WRITE + value: "{{ .Values.connectivity.config.policiesEnforcer.cache.expireAfterWrite }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_EXPIRE_AFTER_ACCESS + value: "{{ .Values.connectivity.config.policiesEnforcer.cache.expireAfterAccess }}" + - name: RECONNECT_RATE_FREQUENCY + value: "{{ .Values.connectivity.config.connections.reconnect.rate.frequency }}" + - name: RECONNECT_RATE_ENTITIES + value: "{{ .Values.connectivity.config.connections.reconnect.rate.entities }}" + - name: CONNECTIVITY_CONNECTION_ALLOWED_HOSTNAMES + value: "{{ .Values.connectivity.config.connections.allowedHostnames }}" + - name: CONNECTIVITY_CONNECTION_BLOCKED_HOSTNAMES + value: "{{ .Values.connectivity.config.connections.blockedHostnames }}" + - name: CONNECTIVITY_CONNECTION_BLOCKED_SUBNETS + value: "{{ .Values.connectivity.config.connections.blockedSubnets }}" + - name: CONNECTIVITY_CONNECTION_BLOCKED_HOST_REGEX + value: "{{ .Values.connectivity.config.connections.blockedHostRegex }}" + - name: CONNECTION_SOURCE_NUMBER + value: "{{ .Values.connectivity.config.connections.limits.maxSources }}" + - name: CONNECTION_TARGET_NUMBER + value: "{{ .Values.connectivity.config.connections.limits.maxTargets }}" + - name: CONNECTIVITY_SIGNAL_ENRICHMENT_BUFFER_SIZE + value: "{{ .Values.connectivity.config.connections.enrichment.bufferSize }}" + - name: KAFKA_CONSUMER_THROTTLING_ENABLED + value: "{{ .Values.connectivity.config.connections.kafka.consumer.throttling.enabled }}" + - name: KAFKA_CONSUMER_THROTTLING_INTERVAL + value: "{{ .Values.connectivity.config.connections.kafka.consumer.throttling.interval }}" + - name: KAFKA_CONSUMER_THROTTLING_LIMIT + value: "{{ .Values.connectivity.config.connections.kafka.consumer.throttling.limit }}" + - name: KAFKA_CONSUMER_THROTTLING_MAX_IN_FLIGHT_FACTOR + value: "{{ .Values.connectivity.config.connections.kafka.consumer.throttling.maxInflightFactor }}" + - name: KAFKA_PRODUCER_QUEUE_SIZE + value: "{{ .Values.connectivity.config.connections.kafka.producer.queueSize }}" + - name: KAFKA_PRODUCER_PARALLELISM + value: "{{ .Values.connectivity.config.connections.kafka.producer.parallelism }}" + {{- if .Values.connectivity.extraEnv }} + {{- toYaml .Values.connectivity.extraEnv | nindent 12 }} + {{- end }} + ports: + - name: remoting + containerPort: {{ .Values.akka.remoting.port }} + protocol: TCP + - name: management + containerPort: {{ .Values.akka.mgmthttp.port }} + protocol: TCP + {{- if .Values.global.prometheus.enabled }} + - name: prometheus + protocol: TCP + containerPort: {{ .Values.global.prometheus.port }} + {{- end }} + readinessProbe: + httpGet: + port: management + path: /ready + initialDelaySeconds: {{ .Values.connectivity.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.connectivity.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.connectivity.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.connectivity.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.connectivity.readinessProbe.failureThreshold }} + livenessProbe: + httpGet: + port: management + path: /alive + initialDelaySeconds: {{ .Values.connectivity.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.connectivity.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.connectivity.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.connectivity.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.connectivity.livenessProbe.failureThreshold }} + volumeMounts: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + mountPath: /opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + subPath: {{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + resources: + requests: + cpu: {{ mulf .Values.connectivity.resources.cpu 1000 }}m + memory: {{ .Values.connectivity.resources.memoryMi }}Mi + limits: + # ## no cpu limit to avoid CFS scheduler limits + # ref: https://doc.akka.io/docs/akka/snapshot/additional/deploy.html#in-kubernetes + # cpu: "" + memory: {{ .Values.connectivity.resources.memoryMi }}Mi + {{- if .Values.openshift.enabled }} + {{- with .Values.openshift.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + {{- end }} + {{- with .Values.connectivity.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.connectivity.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.connectivity.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + configMap: + name: {{ .Release.Name }}-logback-config-connectivity-xml + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + hostPath: + path: /var/log/ditto + type: DirectoryOrCreate + {{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/connectivity-pdb.yaml b/deployment/helm/ditto/templates/connectivity-pdb.yaml new file mode 100644 index 00000000000..c33ebbdee52 --- /dev/null +++ b/deployment/helm/ditto/templates/connectivity-pdb.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.connectivity.podDisruptionBudget.enabled (gt .Values.connectivity.replicaCount 1.0) -}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "ditto.fullname" . }}-connectivity + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity +{{ include "ditto.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity + app.kubernetes.io/instance: {{ .Release.Name }} + minAvailable: {{ .Values.connectivity.podDisruptionBudget.minAvailable }} +{{- end }} diff --git a/deployment/helm/ditto/templates/connectivity-podmonitor.yaml b/deployment/helm/ditto/templates/connectivity-podmonitor.yaml new file mode 100644 index 00000000000..d133f138215 --- /dev/null +++ b/deployment/helm/ditto/templates/connectivity-podmonitor.yaml @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.connectivity.podMonitor.enabled .Values.global.prometheus.port -}} +{{- if .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" -}} +--- +kind: PodMonitor +apiVersion: monitoring.coreos.com/v1 +metadata: + name: {{ include "ditto.fullname" . }}-connectivity + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity +{{ include "ditto.labels" . | indent 4 }} +spec: + podMetricsEndpoints: + - targetPort: {{ .Values.global.prometheus.port }} + path: "/" + {{- if .Values.connectivity.podMonitor.interval }} + interval: {{ .Values.connectivity.podMonitor.interval }} + {{- end }} + {{- if .Values.connectivity.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.connectivity.podMonitor.scrapeTimeout }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity + namespaceSelector: + matchNames: + - {{ $.Release.Namespace | quote }} +{{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/dittoui-config.yaml b/deployment/helm/ditto/templates/dittoui-config.yaml new file mode 100644 index 00000000000..bfba14c9b74 --- /dev/null +++ b/deployment/helm/ditto/templates/dittoui-config.yaml @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.dittoui.enabled -}} +{{- $releaseName := .Release.Name -}} +{{- $name := include "ditto.name" . -}} +{{- $labels := include "ditto.labels" . -}} +{{ $root := . }} +{{ range $path, $bytes := .Files.Glob "dittoui-config/**" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $releaseName }}-{{ $path | replace "/" "-" | replace "." "-" }} + labels: + app.kubernetes.io/name: {{ $name }}-dittoui-config +{{ $labels | indent 4 }} +data: + {{ $path | replace "dittoui-config/" ""}}: |- +{{ $root.Files.Get $path | indent 4 }} +--- +{{- end -}} +{{- end -}} diff --git a/deployment/helm/ditto/templates/dittoui-deployment.yaml b/deployment/helm/ditto/templates/dittoui-deployment.yaml new file mode 100644 index 00000000000..3ba80f3927e --- /dev/null +++ b/deployment/helm/ditto/templates/dittoui-deployment.yaml @@ -0,0 +1,74 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.dittoui.enabled -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ditto.fullname" . }}-dittoui + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui +{{ include "ditto.labels" . | indent 4 }} +spec: + replicas: {{ .Values.dittoui.replicaCount }} + strategy: + {{- with .Values.dittoui.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.dittoui.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.dittoui.additionalAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }}-dittoui + image: {{ printf "%s:%s" .Values.dittoui.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.dittoui.image.tag ) ) }} + imagePullPolicy: {{ .Values.dittoui.image.pullPolicy }} + env: + {{- if .Values.dittoui.extraEnv }} + {{- toYaml .Values.dittoui.extraEnv | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + resources: + requests: + cpu: {{ mulf .Values.dittoui.resources.cpu 1000 }}m + memory: {{ .Values.dittoui.resources.memoryMi }}Mi + limits: + # cpu: "" + memory: {{ .Values.dittoui.resources.memoryMi }}Mi + volumeMounts: + - name: dittoui-nginx-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + volumes: + - name: dittoui-nginx-conf + configMap: + name: {{ .Release.Name }}-dittoui-config-nginx-conf +{{- end }} diff --git a/deployment/helm/ditto/templates/dittoui-pdb.yaml b/deployment/helm/ditto/templates/dittoui-pdb.yaml new file mode 100644 index 00000000000..e3a5a6e52bf --- /dev/null +++ b/deployment/helm/ditto/templates/dittoui-pdb.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.dittoui.podDisruptionBudget.enabled (gt .Values.dittoui.replicaCount 1.0) -}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "ditto.fullname" . }}-dittoui + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui +{{ include "ditto.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui + app.kubernetes.io/instance: {{ .Release.Name }} + minAvailable: {{ .Values.dittoui.podDisruptionBudget.minAvailable }} +{{- end }} diff --git a/deployment/helm/ditto/templates/dittoui-service.yaml b/deployment/helm/ditto/templates/dittoui-service.yaml new file mode 100644 index 00000000000..f640c4a8c15 --- /dev/null +++ b/deployment/helm/ditto/templates/dittoui-service.yaml @@ -0,0 +1,33 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.dittoui.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ditto.fullname" . }}-dittoui + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui +{{ include "ditto.labels" . | indent 4 }} + {{- with .Values.dittoui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ports: + - port: {{ .Values.dittoui.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deployment/helm/ditto/templates/gateway-deployment.yaml b/deployment/helm/ditto/templates/gateway-deployment.yaml new file mode 100644 index 00000000000..51f6536124d --- /dev/null +++ b/deployment/helm/ditto/templates/gateway-deployment.yaml @@ -0,0 +1,293 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.gateway.enabled -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ditto.fullname" . }}-gateway + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway +{{ include "ditto.labels" . | indent 4 }} +spec: + replicas: {{ .Values.gateway.replicaCount }} + strategy: + {{- with .Values.gateway.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.gateway.minReadySeconds }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway + app.kubernetes.io/instance: {{ .Release.Name }} + actorSystemName: {{ .Values.akka.actorSystemName }} + {{- with .Values.gateway.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.global.prometheus.enabled }} + prometheus.io/scrape: "true" + prometheus.io/path: "{{ .Values.global.prometheus.path }}" + prometheus.io/port: "{{ .Values.global.prometheus.port }}" + {{- end }} + checksum/password-config: {{ include (print $.Template.BasePath "/gateway-secret.yaml") . | sha256sum }} + checksum/mongodb-config: {{ include (print $.Template.BasePath "/mongodb-secret.yaml") . | sha256sum }} + {{- with .Values.gateway.additionalAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.rbac.enabled }} + serviceAccountName: {{ template "ditto.serviceAccountName" . }} + {{- end }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + fsGroup: 1000 + initContainers: + {{- if .Values.global.logging.logFiles.enabled }} + - name: change-volume-owner + image: busybox + securityContext: + runAsUser: 0 + command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ] + volumeMounts: + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + containers: + - name: {{ .Chart.Name }}-gateway + image: {{ printf "%s:%s" .Values.gateway.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.gateway.image.tag ) ) }} + imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} + env: + {{- if not .Values.global.logging.customConfigFile.enabled }} + - name: DITTO_LOGGING_DISABLE_SYSOUT_LOG + value: "{{ if .Values.global.logging.sysout.enabled }}false{{ else }}true{{ end }}" + - name: DITTO_LOGGING_FILE_APPENDER + value: "{{ if .Values.global.logging.logFiles.enabled }}true{{ else }}false{{ end }}" + {{- end }} + - name: DITTO_TRACING_ENABLED + value: "{{ .Values.global.tracing.enabled }}" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "{{ .Values.global.tracing.otelExporterOtlpEndpoint }}" + - name: DITTO_TRACING_SAMPLER + value: "{{ .Values.global.tracing.sampler }}" + - name: DITTO_TRACING_RANDOM_SAMPLER_PROBABILITY + value: "{{ .Values.global.tracing.randomSampler.probability }}" + - name: DITTO_TRACING_ADAPTIVE_SAMPLER_THROUGHPUT + value: "{{ .Values.global.tracing.adaptiveSampler.throughput }}" + {{- if .Values.global.logging.logstash.enabled }} + - name: DITTO_LOGGING_LOGSTASH_SERVER + value: "{{ .Values.global.logging.logstash.endpoint }}" + {{- end }} + - name: POD_LABEL_SELECTOR + value: "app.kubernetes.io/name=%s" + - name: POD_NAMESPACE + value: {{ .Release.Namespace }} + - name: INSTANCE_INDEX + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: DISCOVERY_METHOD + value: "kubernetes-api" + - name: TZ + value: "{{ .Values.global.timezone }}" + - name: JAVA_TOOL_OPTIONS + value: > + {{ .Values.global.jvmOptions }} + -XX:ActiveProcessorCount={{ .Values.gateway.jvm.activeProcessorCount }} + -XX:MaxRAMPercentage={{ .Values.gateway.jvm.heapRamPercentage }} + -XX:InitialRAMPercentage={{ .Values.gateway.jvm.heapRamPercentage }} + -XX:MaxGCPauseMillis={{ .Values.gateway.jvm.maxGcPauseMillis }} + {{ .Values.gateway.additionalJvmOptions }} + {{- .Values.global.akkaOptions }} + {{- if .Values.global.logging.customConfigFile.enabled }} + -Dlogback.configurationFile=/opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{- range $key, $value := .Values.gateway.config.authentication.oauth.openidConnectIssuers }} + "{{ printf "%s%s%s=%s" "-Dditto.gateway.authentication.oauth.openid-connect-issuers." $key ".issuer" $value.issuer }}" + {{- range $index, $subject := $value.authSubjects }} + "{{ printf "%s%s%s%d=%s" "-Dditto.gateway.authentication.oauth.openid-connect-issuers." $key ".auth-subjects." $index $subject }}" + {{- end }} + {{- end }} + {{- range $key, $value := .Values.gateway.config.authentication.devops.oauth.openidConnectIssuers }} + "{{ printf "%s%s%s=%s" "-Dditto.gateway.authentication.devops.oauth.openid-connect-issuers." $key ".issuer" $value.issuer }}" + {{- range $index, $subject := $value.authSubjects }} + "{{ printf "%s%s%s%d=%s" "-Dditto.gateway.authentication.devops.oauth.openid-connect-issuers." $key ".auth-subjects." $index $subject }}" + {{- end }} + {{- end }} + {{ join " " .Values.gateway.systemProps }} + - name: CLUSTER_BS_REQUIRED_CONTACTS + value: "{{ .Values.global.cluster.requiredContactPoints }}" + - name: DITTO_DDATA_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.ddata.numberOfShards }}" + - name: DITTO_DDATA_MAX_DELTA_ELEMENTS + value: "{{ .Values.global.cluster.ddata.maxDeltaElements }}" + - name: CLUSTER_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.numberOfShards }}" + - name: CLUSTER_DOWNING_STABLE_AFTER + value: "{{ .Values.global.cluster.downingStableAfter }}" + - name: CLUSTER_DOWNING_DOWN_ALL_WHEN_UNSTABLE + value: "{{ .Values.global.cluster.downAllWhenUnstable }}" + {{- if .Values.global.prometheus.enabled }} + - name: PROMETHEUS_PORT + value: "{{ .Values.global.prometheus.port }}" + {{- end }} + - name: ENABLE_PRE_AUTHENTICATION + value: "{{ .Values.gateway.config.authentication.enablePreAuthentication }}" + - name: DEVOPS_SECURED + value: "{{ .Values.gateway.config.authentication.devops.secured }}" + - name: DEVOPS_AUTHENTICATION_METHOD + value: "{{ .Values.gateway.config.authentication.devops.authMethod }}" + - name: DEVOPS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.gateway.config.authentication.devops.existingSecret | default ( printf "%s-gateway-secret" ( include "ditto.fullname" . )) }} + key: devops-password + {{- range $index, $oauthSubject := .Values.gateway.config.authentication.devops.oauthSubjects }} + - name: DEVOPS_OAUTH2_SUBJECTS.{{ $index }} + value: "{{ $oauthSubject }}" + {{- end }} + - name: DEVOPS_STATUS_SECURED + value: "{{ .Values.gateway.config.authentication.devops.statusSecured }}" + - name: STATUS_AUTHENTICATION_METHOD + value: "{{ .Values.gateway.config.authentication.devops.statusAuthMethod }}" + - name: STATUS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.gateway.config.authentication.devops.existingSecret | default ( printf "%s-gateway-secret" ( include "ditto.fullname" . )) }} + key: status-password + {{- range $index, $oauthSubject := .Values.gateway.config.authentication.devops.statusOauthSubjects }} + - name: STATUS_OAUTH2_SUBJECTS.{{ $index }} + value: "{{ $oauthSubject }}" + {{- end }} + - name: WS_SUBSCRIBER_BACKPRESSURE + value: "{{ .Values.gateway.config.websocket.subscriber.backpressureQueueSize }}" + - name: WS_PUBLISHER_BACKPRESSURE + value: "{{ .Values.gateway.config.websocket.publisher.backpressureBufferSize }}" + - name: GATEWAY_WEBSOCKET_THROTTLING_ENABLED + value: "{{ .Values.gateway.config.websocket.throttling.enabled }}" + - name: GATEWAY_WEBSOCKET_THROTTLING_INTERVAL + value: "{{ .Values.gateway.config.websocket.throttling.interval }}" + - name: GATEWAY_WEBSOCKET_THROTTLING_LIMIT + value: "{{ .Values.gateway.config.websocket.throttling.limit }}" + - name: GATEWAY_SSE_THROTTLING_ENABLED + value: "{{ .Values.gateway.config.sse.throttling.enabled }}" + - name: GATEWAY_SSE_THROTTLING_INTERVAL + value: "{{ .Values.gateway.config.sse.throttling.interval }}" + - name: GATEWAY_SSE_THROTTLING_LIMIT + value: "{{ .Values.gateway.config.sse.throttling.limit }}" + - name: OAUTH_ALLOWED_CLOCK_SKEW + value: "{{ .Values.gateway.config.authentication.oauth.allowedClockSkew }}" + {{- if .Values.gateway.extraEnv }} + {{- toYaml .Values.gateway.extraEnv | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.gateway.service.port }} + protocol: TCP + - name: remoting + containerPort: {{ .Values.akka.remoting.port }} + protocol: TCP + - name: management + containerPort: {{ .Values.akka.mgmthttp.port }} + protocol: TCP + {{- if .Values.global.prometheus.enabled }} + - name: prometheus + protocol: TCP + containerPort: {{ .Values.global.prometheus.port }} + {{- end }} + readinessProbe: + httpGet: + port: management + path: /ready + initialDelaySeconds: {{ .Values.gateway.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.gateway.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.gateway.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.gateway.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.gateway.readinessProbe.failureThreshold }} + livenessProbe: + httpGet: + port: management + path: /alive + initialDelaySeconds: {{ .Values.gateway.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.gateway.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.gateway.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.gateway.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.gateway.livenessProbe.failureThreshold }} + volumeMounts: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + mountPath: /opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + subPath: {{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + resources: + requests: + cpu: {{ mulf .Values.gateway.resources.cpu 1000 }}m + memory: {{ .Values.gateway.resources.memoryMi }}Mi + limits: + # ## no cpu limit to avoid CFS scheduler limits + # ref: https://doc.akka.io/docs/akka/snapshot/additional/deploy.html#in-kubernetes + # cpu: "" + memory: {{ .Values.gateway.resources.memoryMi }}Mi + {{- if .Values.openshift.enabled }} + {{- with .Values.openshift.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + {{- end }} + {{- with .Values.gateway.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + configMap: + name: {{ .Release.Name }}-logback-config-gateway-xml + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + hostPath: + path: /var/log/ditto + type: DirectoryOrCreate + {{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/gateway-pdb.yaml b/deployment/helm/ditto/templates/gateway-pdb.yaml new file mode 100644 index 00000000000..1be894d9e5f --- /dev/null +++ b/deployment/helm/ditto/templates/gateway-pdb.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.gateway.podDisruptionBudget.enabled (gt .Values.gateway.replicaCount 1.0) -}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "ditto.fullname" . }}-gateway + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway +{{ include "ditto.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway + app.kubernetes.io/instance: {{ .Release.Name }} + minAvailable: {{ .Values.gateway.podDisruptionBudget.minAvailable }} +{{- end }} diff --git a/deployment/helm/ditto/templates/gateway-podmonitor.yaml b/deployment/helm/ditto/templates/gateway-podmonitor.yaml new file mode 100644 index 00000000000..b57a9b65de4 --- /dev/null +++ b/deployment/helm/ditto/templates/gateway-podmonitor.yaml @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.gateway.podMonitor.enabled .Values.global.prometheus.port -}} +{{- if .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" -}} +--- +kind: PodMonitor +apiVersion: monitoring.coreos.com/v1 +metadata: + name: {{ include "ditto.fullname" . }}-gateway + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway +{{ include "ditto.labels" . | indent 4 }} +spec: + podMetricsEndpoints: + - targetPort: {{ .Values.global.prometheus.port }} + path: "/" + {{- if .Values.gateway.podMonitor.interval }} + interval: {{ .Values.gateway.podMonitor.interval }} + {{- end }} + {{- if .Values.gateway.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.gateway.podMonitor.scrapeTimeout }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway + namespaceSelector: + matchNames: + - {{ $.Release.Namespace | quote }} +{{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/gateway-secret.yaml b/deployment/helm/ditto/templates/gateway-secret.yaml new file mode 100644 index 00000000000..45e9a42c6af --- /dev/null +++ b/deployment/helm/ditto/templates/gateway-secret.yaml @@ -0,0 +1,32 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if not .Values.gateway.config.authentication.devops.existingSecret }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ditto.fullname" . }}-gateway-secret + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway-secret +{{ include "ditto.labels" . | indent 4 }} +type: Opaque +data: + {{- if .Values.gateway.config.authentication.devops.devopsPassword }} + devops-password: {{ .Values.gateway.config.authentication.devops.devopsPassword | b64enc | quote }} + {{- else }} + devops-password: {{ randAlphaNum 12 | b64enc | quote }} + {{- end }} + {{- if .Values.gateway.config.authentication.devops.statusPassword }} + status-password: {{ .Values.gateway.config.authentication.devops.statusPassword | b64enc | quote }} + {{- else }} + status-password: {{ randAlphaNum 12 | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/gateway-service.yaml b/deployment/helm/ditto/templates/gateway-service.yaml new file mode 100644 index 00000000000..c4549ee7c00 --- /dev/null +++ b/deployment/helm/ditto/templates/gateway-service.yaml @@ -0,0 +1,33 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.gateway.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ditto.fullname" . }}-gateway + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway +{{ include "ditto.labels" . | indent 4 }} + {{- with .Values.gateway.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ports: + - port: {{ .Values.gateway.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deployment/helm/ditto/templates/logback-config.yaml b/deployment/helm/ditto/templates/logback-config.yaml new file mode 100644 index 00000000000..029cf98467a --- /dev/null +++ b/deployment/helm/ditto/templates/logback-config.yaml @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.global.logging.customConfigFile.enabled -}} +{{- $releaseName := .Release.Name -}} +{{- $name := include "ditto.name" . -}} +{{- $labels := include "ditto.labels" . -}} +{{- $logbackFileName := .Values.global.logging.customConfigFile.fileName -}} +{{ $root := . }} +{{ range $path, $bytes := .Files.Glob "logback-config/**" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $releaseName }}-{{ $path | replace "/" "-" | replace "." "-" }} + labels: + app.kubernetes.io/name: {{ $name }}-logback-config +{{ $labels | indent 4 }} +data: + {{ $logbackFileName }}: |- +{{ $root.Files.Get $path | indent 4 }} +--- +{{- end -}} +{{- end -}} diff --git a/deployment/helm/ditto/templates/mongodb-secret.yaml b/deployment/helm/ditto/templates/mongodb-secret.yaml new file mode 100644 index 00000000000..06584e605e5 --- /dev/null +++ b/deployment/helm/ditto/templates/mongodb-secret.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if not .Values.dbconfig.uriSecret -}} +{{- $mongoName := include "ditto.mongodb.fullname" . -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ditto.fullname" . }}-mongodb-secret + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-mongodb-secret +{{ include "ditto.labels" . | indent 4 }} +type: Opaque +data: + connectivity-uri: {{ .Values.dbconfig.connectivity.uri | replace "#{PLACEHOLDER_MONGODB_HOSTNAME}#" $mongoName | b64enc | quote}} + things-uri: {{ .Values.dbconfig.things.uri | replace "#{PLACEHOLDER_MONGODB_HOSTNAME}#" $mongoName | b64enc | quote}} + thingsSearch-uri: {{ .Values.dbconfig.thingsSearch.uri | replace "#{PLACEHOLDER_MONGODB_HOSTNAME}#" $mongoName | b64enc | quote}} + policies-uri: {{ .Values.dbconfig.policies.uri | replace "#{PLACEHOLDER_MONGODB_HOSTNAME}#" $mongoName | b64enc | quote}} +{{- end }} diff --git a/deployment/helm/ditto/templates/nginx-auth.yaml b/deployment/helm/ditto/templates/nginx-auth.yaml new file mode 100644 index 00000000000..582cf89e8c5 --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-auth.yaml @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.nginx.enabled -}} +{{- $releaseName := .Release.Name -}} +{{- $name := include "ditto.name" . -}} +{{- $labels := include "ditto.labels" . -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $releaseName }}-nginx-config-nginx-htpasswd + labels: + app.kubernetes.io/name: {{ $name }}-nginx-config +{{ $labels | indent 4 }} +type: Opaque +stringData: + nginx.htpasswd: |- +{{- if .Values.global.hashedBasicAuthUsers }} +{{ range .Values.global.hashedBasicAuthUsers }} +{{- . | indent 4 }} +{{ end }} +{{- else }} +{{ range $key, $value := .Values.global.basicAuthUsers }} +{{- (htpasswd $value.user $value.password) | indent 4 }} +{{ end }} +{{ end }} +--- +{{- end }} diff --git a/deployment/helm/ditto/templates/nginx-config.yaml b/deployment/helm/ditto/templates/nginx-config.yaml new file mode 100644 index 00000000000..0d6e51a47f4 --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-config.yaml @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.nginx.enabled -}} +{{- $releaseName := .Release.Name -}} +{{- $name := include "ditto.name" . -}} +{{- $labels := include "ditto.labels" . -}} +{{ $root := . }} +{{ range $path, $bytes := .Files.Glob "nginx-config/**" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $releaseName }}-{{ $path | replace "/" "-" | replace "." "-" }} + labels: + app.kubernetes.io/name: {{ $name }}-nginx-config +{{ $labels | indent 4 }} +data: + {{ $path | replace "nginx-config/" ""}}: |- +{{ $root.Files.Get $path | indent 4 }} +--- +{{- end -}} +{{- end -}} diff --git a/deployment/helm/ditto/templates/nginx-configmap.yaml b/deployment/helm/ditto/templates/nginx-configmap.yaml new file mode 100644 index 00000000000..b9667b661ed --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-configmap.yaml @@ -0,0 +1,242 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.nginx.enabled -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-nginx-conf + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx-conf +{{ include "ditto.labels" . | indent 4 }} +data: + nginx.conf: |- + worker_processes {{ .Values.nginx.config.workerProcesses }}; + + events { + worker_connections {{ .Values.nginx.config.workerConnections }}; + } + + http { + charset utf-8; + default_type application/json; + include mime.types; + + # timeouts are configured slightly higher than ditto-eclipse-ditto-gateway read-timeout of 60 seconds + proxy_connect_timeout 70; # seconds, default: 60 + proxy_send_timeout 70; # seconds, default: 60 + proxy_read_timeout 70; # seconds, default: 60 + # will try another upstream if an error or timeout occurred during the connection + # or if the upstream returns 502 response + proxy_next_upstream error timeout http_502; + # will retry up to 3 times to find another upstream to connect to + proxy_next_upstream_tries 3; + # will try for max. 20s to find another upstream to connect to + proxy_next_upstream_timeout 20; + + send_timeout 70; # seconds, default: 60 + + client_header_buffer_size 8k; # allow longer URIs + headers (default: 1k) + large_client_header_buffers 4 16k; + + merge_slashes off; # allow multiple slashes for CRS Authentication + + map $http_authorization $authentication { + default "Authentication required"; + "~Bearer" "off"; + # the above means: if we get a request containing an "Authorization: Bearer ..." header, set "off" to $authentication + } + + map $http_authorization $nginx_auth_user { + default "nginx:${remote_user}"; + "~Bearer" ""; + } + + upstream {{ include "ditto.fullname" . }}-gateway { + server {{ include "ditto.fullname" . }}-gateway:8080; + } + + {{ if .Values.dittoui.enabled -}} + upstream {{ include "ditto.fullname" . }}-dittoui { + server {{ include "ditto.fullname" . }}-dittoui:8080; + } + {{- end }} + + {{ if .Values.swaggerui.enabled -}} + upstream {{ include "ditto.fullname" . }}-swaggerui { + server {{ include "ditto.fullname" . }}-swaggerui:8080; + } + {{- end }} + + log_format jsonlog escape=json '{' + '"@timestamp":"$time_iso8601",' + '"remote_addr":"$remote_addr",' + '"remote_user":"$remote_user",' + '"request":"$request",' + '"status": "$status",' + '"body_bytes_sent":"$body_bytes_sent",' + '"request_time":"$request_time",' + '"upstream_response_time":"$upstream_response_time",' + '"http_referrer":"$http_referer",' + '"http_user_agent":"$http_user_agent",' + '"correlation-id":"$http_correlation_id"' + '}'; + access_log /var/log/nginx/access.log jsonlog; + + server { + listen 8080; + server_name localhost; + + location / { + index index.html; + } + + # api + location /api { + include nginx-cors.conf; + + {{ if .Values.global.jwtOnly -}} + proxy_pass_request_headers on; + proxy_set_header Authorization $http_authorization; + {{ else }} + auth_basic $authentication; + auth_basic_user_file nginx.htpasswd; + proxy_set_header X-Forwarded-User $remote_user; + proxy_set_header x-ditto-pre-authenticated $nginx_auth_user; + {{- end }} + + proxy_pass http://{{ include "ditto.fullname" . }}-gateway; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_set_header Connection ''; + chunked_transfer_encoding off; + proxy_buffering off; + proxy_cache off; + } + + # ws + location /ws { + + {{ if .Values.global.jwtOnly -}} + proxy_pass_request_headers on; + proxy_set_header Authorization $http_authorization; + {{ else }} + auth_basic $authentication; + auth_basic_user_file nginx.htpasswd; + proxy_set_header X-Forwarded-User $remote_user; + proxy_set_header x-ditto-pre-authenticated $nginx_auth_user; + {{- end }} + + proxy_pass http://{{ include "ditto.fullname" . }}-gateway; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 1d; + proxy_send_timeout 1d; + } + + # health + location /health { + include nginx-cors.conf; + + # exclude health checks from being logged in access log: + access_log off; + + proxy_pass http://{{ include "ditto.fullname" . }}-gateway/health; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-User $remote_user; + } + + # status + location /status { + include nginx-cors.conf; + + # exclude status access from being logged in access log: + access_log off; + + proxy_pass http://{{ include "ditto.fullname" . }}-gateway/overall/status; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-User $remote_user; + } + + # stats + location /stats { + include nginx-cors.conf; + + # exclude stats access from being logged in access log: + access_log off; + + proxy_pass http://{{ include "ditto.fullname" . }}-gateway/stats; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-User $remote_user; + } + + # devops + location /devops { + include nginx-cors.conf; + + proxy_pass http://{{ include "ditto.fullname" . }}-gateway/devops; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-User $remote_user; + } + + # connections api using devops user configured in Ditto + location /api/2/connections { + include nginx-cors.conf; + + proxy_pass http://{{ include "ditto.fullname" . }}-gateway; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-User $remote_user; + } + + {{ if .Values.dittoui.enabled -}} + location /ui/ { + proxy_pass http://{{ include "ditto.fullname" . }}-dittoui/; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + } + {{- end }} + + {{ if .Values.swaggerui.enabled -}} + # swagger + # access API doc on: /apidoc/ + location /apidoc/ { + proxy_pass http://{{ include "ditto.fullname" . }}-swaggerui/; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + } + {{- end }} + } + } + +{{- end }} diff --git a/deployment/helm/ditto/templates/nginx-deployment.yaml b/deployment/helm/ditto/templates/nginx-deployment.yaml new file mode 100644 index 00000000000..3fa88faec5a --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-deployment.yaml @@ -0,0 +1,149 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.nginx.enabled -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ditto.fullname" . }}-nginx + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx +{{ include "ditto.labels" . | indent 4 }} +spec: + replicas: {{ .Values.nginx.replicaCount }} + strategy: + {{- with .Values.nginx.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.nginx.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/nginx-conf: {{ include (print $.Template.BasePath "/nginx-configmap.yaml") . | sha256sum }} + checksum/nginx-config: {{ include (print $.Template.BasePath "/nginx-config.yaml") . | sha256sum }} + checksum/nginx-auth: {{ include (print $.Template.BasePath "/nginx-auth.yaml") . | sha256sum }} + {{- with .Values.nginx.additionalAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.nginx.initContainers.waitForGateway.enabled }} + initContainers: + - name: {{ .Values.nginx.initContainers.waitForGateway.name }} + image: {{ .Values.nginx.initContainers.waitForGateway.image }} + args: + - /bin/sh + - -c + - > + set -x; + while [[ "$(curl -sL -w "%{http_code}\n" http://{{ include "ditto.fullname" . }}-gateway:8080/health -o /dev/null)" != "200" ]]; do + echo '.' + sleep 1; + done + {{- end }} + containers: + - name: {{ .Chart.Name }}-nginx + image: "{{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag }}" + imagePullPolicy: {{ .Values.nginx.image.pullPolicy }} + env: + {{- if .Values.nginx.extraEnv }} + {{- toYaml .Values.nginx.extraEnv | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + {{- if .Values.nginx.readinessProbe }} + readinessProbe: + {{- toYaml .Values.nginx.readinessProbe | nindent 12 }} + {{- end }} + {{- if .Values.nginx.livenessProbe }} + livenessProbe: + {{- toYaml .Values.nginx.livenessProbe | nindent 12 }} + {{- end }} + resources: + requests: + cpu: {{ mulf .Values.nginx.resources.cpu 1000 }}m + memory: {{ .Values.nginx.resources.memoryMi }}Mi + limits: + # cpu: "" + memory: {{ .Values.nginx.resources.memoryMi }}Mi + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: nginx-htpasswd + mountPath: /etc/nginx/nginx.htpasswd + subPath: nginx.htpasswd + - name: nginx-cors + mountPath: /etc/nginx/nginx-cors.conf + subPath: nginx-cors.conf + - name: nginx-index + mountPath: /etc/nginx/html/index.html + subPath: index.html + - name: nginx-ditto-down + mountPath: /etc/nginx/html/ditto-down.svg + subPath: ditto-down.svg + - name: nginx-ditto-up + mountPath: /etc/nginx/html/ditto-up.svg + subPath: ditto-up.svg + - name: nginx-cache + mountPath: /var/cache/nginx + - name: nginx-run + mountPath: /run/nginx + volumes: + - name: nginx-conf + configMap: + name: {{ .Release.Name }}-nginx-conf + - name: nginx-htpasswd + secret: + secretName: {{ .Release.Name }}-nginx-config-nginx-htpasswd + - name: nginx-cors + configMap: + name: {{ .Release.Name }}-nginx-config-nginx-cors-conf + - name: nginx-index + configMap: + name: {{ .Release.Name }}-nginx-config-index-html + - name: nginx-ditto-down + configMap: + name: {{ .Release.Name }}-nginx-config-ditto-down-svg + - name: nginx-ditto-up + configMap: + name: {{ .Release.Name }}-nginx-config-ditto-up-svg + - name: nginx-cache + emptyDir: {} + - name: nginx-run + emptyDir: {} + {{- with .Values.nginx.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nginx.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nginx.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/nginx-ingress-api.yaml b/deployment/helm/ditto/templates/nginx-ingress-api.yaml new file mode 100644 index 00000000000..d1900b7ab8f --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-ingress-api.yaml @@ -0,0 +1,62 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "ditto.fullname" . -}} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }}-api + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx +{{ include "ditto.labels" . | indent 4 }} + annotations: + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.api.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className }} + defaultBackend: + service: + name: {{ $fullName }}-{{ .Values.ingress.defaultBackendSuffix }} + port: + name: http +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + {{- range .Values.ingress.api.paths }} + - path: {{ .path }} + {{- if .pathType }} + pathType: {{ .pathType }} + {{- else }} + pathType: Prefix + {{- end }} + backend: + service: + name: {{ $fullName }}-{{ .backendSuffix }} + port: + name: http + {{- end }} +{{- end }} \ No newline at end of file diff --git a/deployment/helm/ditto/templates/nginx-ingress-root.yaml b/deployment/helm/ditto/templates/nginx-ingress-root.yaml new file mode 100644 index 00000000000..655af5a6160 --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-ingress-root.yaml @@ -0,0 +1,62 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "ditto.fullname" . -}} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }}-root + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx +{{ include "ditto.labels" . | indent 4 }} + annotations: + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.root.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className }} + defaultBackend: + service: + name: {{ $fullName }}-{{ .Values.ingress.defaultBackendSuffix }} + port: + name: http +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + {{- range .Values.ingress.root.paths }} + - path: {{ .path }} + {{- if .pathType }} + pathType: {{ .pathType }} + {{- else }} + pathType: Prefix + {{- end }} + backend: + service: + name: {{ $fullName }}-{{ .backendSuffix }} + port: + name: http + {{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/nginx-ingress-ui.yaml b/deployment/helm/ditto/templates/nginx-ingress-ui.yaml new file mode 100644 index 00000000000..b16d6b324a2 --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-ingress-ui.yaml @@ -0,0 +1,62 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "ditto.fullname" . -}} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }}-ui + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx +{{ include "ditto.labels" . | indent 4 }} + annotations: + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.ui.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className }} + defaultBackend: + service: + name: {{ $fullName }}-{{ .Values.ingress.defaultBackendSuffix }} + port: + name: http +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + {{- range .Values.ingress.ui.paths }} + - path: {{ .path }} + {{- if .pathType }} + pathType: {{ .pathType }} + {{- else }} + pathType: Prefix + {{- end }} + backend: + service: + name: {{ $fullName }}-{{ .backendSuffix }} + port: + name: http + {{- end }} +{{- end }} \ No newline at end of file diff --git a/deployment/helm/ditto/templates/nginx-ingress-ws.yaml b/deployment/helm/ditto/templates/nginx-ingress-ws.yaml new file mode 100644 index 00000000000..d0a425919ef --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-ingress-ws.yaml @@ -0,0 +1,62 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "ditto.fullname" . -}} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }}-ws + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx +{{ include "ditto.labels" . | indent 4 }} + annotations: + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.ws.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className }} + defaultBackend: + service: + name: {{ $fullName }}-{{ .Values.ingress.defaultBackendSuffix }} + port: + name: http +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + {{- range .Values.ingress.ws.paths }} + - path: {{ .path }} + {{- if .pathType }} + pathType: {{ .pathType }} + {{- else }} + pathType: Prefix + {{- end }} + backend: + service: + name: {{ $fullName }}-{{ .backendSuffix }} + port: + name: http + {{- end }} +{{- end }} \ No newline at end of file diff --git a/deployment/helm/ditto/templates/nginx-ingress.yaml b/deployment/helm/ditto/templates/nginx-ingress.yaml new file mode 100644 index 00000000000..75c2f5abd60 --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-ingress.yaml @@ -0,0 +1,843 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.ingress.controller.enabled -}} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: "{{ .Values.ingress.controller.namespace }}" + labels: + name: "{{ .Values.ingress.controller.namespace }}" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-node-health-check-conf + namespace: "{{ .Values.ingress.controller.namespace }}" +data: + nginx-node-health-check.conf: | + # config for health check container running together with the ingress controller + worker_processes 1; + + events { + worker_connections 1024; + } + + http { + charset utf-8; + + server { + listen 8080; + server_name localhost; + + location /healthz { + add_header Content-Type text-plain; + return 200 'ok'; + } + + } + } + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.namespace }}" + name: "{{ .Values.ingress.controller.namespace }}" + namespace: "{{ .Values.ingress.controller.namespace }}" +spec: + selector: + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + ports: + - name: tcp + port: 80 + protocol: TCP + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxIngressVersion }}" + name: nginx-ingress-controller-admission + namespace: "{{ .Values.ingress.controller.namespace }}" +spec: + ports: + - appProtocol: https + name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + type: ClusterIP + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-configuration + namespace: "{{ .Values.ingress.controller.namespace }}" +data: + worker-processes: "auto" + max-worker-connections: "0" # 0 will use the value of max-worker-open-files + max-worker-open-files: "0" # the default of 0 means "max open files (system's limit) / worker-processes - 1024" + server-tokens: "False" + use-gzip: "True" + gzip-level: "6" + gzip-types: "text/plain text/css text/js text/xml text/javascript application/javascript application/x-javascript application/json application/xml application/xml+rss" + location-snippet: | + more_set_headers "Strict-Transport-Security: max-age=31536000; includeSubDomains"; + more_set_headers 'X-Content-Type-Options: "nosniff"'; + more_set_headers 'X-Frame-Options: "SAMEORIGIN"'; + more_set_headers 'X-XSS-Protection: "1; mode=block"'; + default_type application/octet-stream; + gzip_disable "msie6"; + gzip_min_length 1100; + gzip_buffers 16 8k; + gzip_proxied any; + gunzip on; + gzip_static always; + gzip_vary on; + + tcp_nopush on; + + # timeouts are configured slightly higher than gateway read-timeout of 60 seconds + send_timeout 70; # seconds, default: 60 + + # ignore X-Original-URI in the request + proxy_hide_header X-Original-URI; + proxy_set_header X-Original-URI $request_uri; + + # set ditto-specific forwarded headers + + proxy-connect-timeout: "10" # seconds, default: 60 + # timeouts are configured slightly higher than gateway read-timeout of 60 seconds + proxy-send-timeout: "70" # seconds, default: 60 + proxy-read-timeout: "70" # seconds, default: 60 + # will try another upstream if an error or timeout occurred during the connection + # or if the upstream returns 502 response + proxy-next-upstream: "error timeout http_502" + # will retry up to 4 times to find another upstream to connect to + proxy-next-upstream-tries: "4" + # will try for max. 50s to find another upstream to connect to + proxy-next-upstream-timeout: "50" + client-header-buffer-size: "8k" # allow longer URIs + headers (default: 1k) + large-client-header-buffers: "4 16k" + keep-alive: "75" #seconds, default: 75 + log-format-upstream: '$remote_addr - "$remote_user" [$time_local] "$host" "$request" $status $bytes_sent "$upstream_addr" "$http_referer" "$http_user_agent" "$http_origin" "$http_content_type"' + use-forwarded-headers: "True" + http-snippet: | + charset utf-8; + sendfile on; + + # timeouts are configured slightly higher than gateway read-timeout of 60 seconds + send_timeout 70; # seconds, default: 60 + + merge_slashes off; # allow multiple slashes for CRS Authentication + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: tcp-services + namespace: "{{ .Values.ingress.controller.namespace }}" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: udp-services + namespace: "{{ .Values.ingress.controller.namespace }}" + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-serviceaccount + namespace: "{{ .Values.ingress.controller.namespace }}" + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission + namespace: "{{ .Values.ingress.controller.namespace }}" + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-role + namespace: "{{ .Values.ingress.controller.namespace }}" +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resourceNames: + - ingress-controller-leader + resources: + - configmaps + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - coordination.k8s.io + resourceNames: + - ingress-controller-leader + resources: + - leases + verbs: + - get + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission + namespace: "{{ .Values.ingress.controller.namespace }}" +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: "{{ .Values.ingress.controller.namespace }}-clusterrole" +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-role-binding + namespace: "{{ .Values.ingress.controller.namespace }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-ingress-role +subjects: +- kind: ServiceAccount + name: nginx-ingress-serviceaccount + namespace: "{{ .Values.ingress.controller.namespace }}" + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission + namespace: "{{ .Values.ingress.controller.namespace }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-ingress-admission +subjects: + - kind: ServiceAccount + name: nginx-ingress-admission + namespace: "{{ .Values.ingress.controller.namespace }}" + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: "{{ .Values.ingress.controller.namespace }}-clusterrole-binding" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: "{{ .Values.ingress.controller.namespace }}-clusterrole" +subjects: +- kind: ServiceAccount + name: nginx-ingress-serviceaccount + namespace: "{{ .Values.ingress.controller.namespace }}" + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx-ingress-admission +subjects: + - kind: ServiceAccount + name: nginx-ingress-admission + namespace: "{{ .Values.ingress.controller.namespace }}" + +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority +value: 1000000 +globalDefault: false +description: "This priority class is used for the things and gateway service pods only." + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-controller + namespace: "{{ .Values.ingress.controller.namespace }}" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + minReadySeconds: 10 + revisionHistoryLimit: 5 + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + spec: + priorityClassName: high-priority + serviceAccountName: nginx-ingress-serviceaccount + dnsPolicy: ClusterFirst + nodeSelector: + ingress.node: "master" + terminationGracePeriodSeconds: 100 + imagePullSecrets: + - name: acr-secret + securityContext: + fsGroup: 101 + supplementalGroups: [101] + seccompProfile: + type: RuntimeDefault + containers: + - name: nginx-node-health-check + image: docker.io/library/nginx:{{ .Values.ingress.controller.nginxVersion }} + imagePullPolicy: Always + ports: + - name: healthz-port + containerPort: 8080 + hostPort: 31005 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 6 + successThreshold: 1 + failureThreshold: 4 + readinessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + successThreshold: 1 + volumeMounts: + - name: nginx-node-health-check-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx-node-health-check.conf + - name: nginx-ingress-controller + image: registry.k8s.io/ingress-nginx/controller:{{ .Values.ingress.controller.nginxIngressVersion }} + imagePullPolicy: IfNotPresent + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/{{ .Values.ingress.controller.namespace }} + - --election-id=ingress-controller-leader + - --controller-class=k8s.io/{{ .Values.ingress.controller.namespace }} + - --ingress-class={{ .Values.ingress.className }} + - --configmap=$(POD_NAMESPACE)/nginx-configuration + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --shutdown-grace-period=65 + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + # www-data -> 101 + runAsUser: 101 + runAsGroup: 101 + runAsNonRoot: true + allowPrivilegeEscalation: true + seccompProfile: + type: RuntimeDefault + lifecycle: + preStop: + exec: + command: ["sleep", "95"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + ports: + - name: http + containerPort: 80 + hostPort: 30005 + - name: https + containerPort: 443 + protocol: TCP + - name: health + containerPort: 10254 + - name: webhook + containerPort: 8443 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 6 + successThreshold: 1 + failureThreshold: 4 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + successThreshold: 1 + resources: + requests: + cpu: "0.75" + memory: "1024Mi" + volumeMounts: + - mountPath: /usr/local/certificates/ + name: webhook-cert + readOnly: true + volumes: + - name: nginx-node-health-check-conf + configMap: + name: nginx-node-health-check-conf + - name: webhook-cert + secret: + secretName: nginx-ingress-admission +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission-create + namespace: "{{ .Values.ingress.controller.namespace }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission-create + spec: + containers: + - args: + - create + - --host=nginx-ingress-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=nginx-ingress-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f + imagePullPolicy: IfNotPresent + name: create + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: nginx-ingress-admission + +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission-patch + namespace: "{{ .Values.ingress.controller.namespace }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission-patch + spec: + containers: + - args: + - patch + - --webhook-name=nginx-ingress-admission-{{ .Values.ingress.controller.namespace }} + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=nginx-ingress-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f + imagePullPolicy: IfNotPresent + name: patch + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: nginx-ingress-admission + +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: "{{ .Values.ingress.className }}" +spec: + controller: k8s.io/{{ .Values.ingress.controller.namespace }} + +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" + app.kubernetes.io/version: "{{ .Values.ingress.controller.nginxVersion }}" + name: nginx-ingress-admission-{{ .Values.ingress.controller.namespace }} +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: nginx-ingress-controller-admission + namespace: "{{ .Values.ingress.controller.namespace }}" + path: /networking/v1/ingresses + failurePolicy: Fail + matchPolicy: Equivalent + name: validate.nginx.ingress.kubernetes.io + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + sideEffects: None +{{- end }} diff --git a/deployment/helm/ditto/templates/nginx-route.yaml b/deployment/helm/ditto/templates/nginx-route.yaml new file mode 100644 index 00000000000..dff36831c6b --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-route.yaml @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.openshift.routes.enabled -}} +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: {{ include "ditto.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx +{{ include "ditto.labels" . | indent 4 }} + {{- with .Values.openshift.routes.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + host: {{ .Values.openshift.routes.host | default "" }} + to: + kind: Service + name: {{ include "ditto.fullname" . }}-nginx + weight: 100 + port: + targetPort: {{ .Values.openshift.routes.targetPort }} + tls: + termination: {{ .Values.openshift.routes.tlsTermination | default "edge" }} + insecureEdgeTerminationPolicy: {{ .Values.openshift.routes.tlsInsecurePolicy | default "Redirect" }} +{{- end }} diff --git a/deployment/helm/ditto/templates/nginx-service.yaml b/deployment/helm/ditto/templates/nginx-service.yaml new file mode 100644 index 00000000000..7e711056315 --- /dev/null +++ b/deployment/helm/ditto/templates/nginx-service.yaml @@ -0,0 +1,37 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.nginx.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ditto.fullname" . }}-nginx + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx +{{ include "ditto.labels" . | indent 4 }} + {{- with .Values.nginx.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.nginx.service.type }} + ports: + - port: {{ .Values.nginx.service.port }} + {{- if (and (eq .Values.nginx.service.type "NodePort") (not (empty .Values.nginx.service.nodePort))) }} + nodePort: {{ .Values.nginx.service.nodePort }} + {{- end }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deployment/helm/ditto/templates/policies-deployment.yaml b/deployment/helm/ditto/templates/policies-deployment.yaml new file mode 100644 index 00000000000..ea372d7915e --- /dev/null +++ b/deployment/helm/ditto/templates/policies-deployment.yaml @@ -0,0 +1,310 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.policies.enabled -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ditto.fullname" . }}-policies + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-policies +{{ include "ditto.labels" . | indent 4 }} +spec: + replicas: {{ .Values.policies.replicaCount }} + strategy: + {{- with .Values.policies.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.policies.minReadySeconds }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-policies + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-policies + app.kubernetes.io/instance: {{ .Release.Name }} + actorSystemName: {{ .Values.akka.actorSystemName }} + {{- with .Values.policies.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.global.prometheus.enabled }} + prometheus.io/scrape: "true" + prometheus.io/path: "{{ .Values.global.prometheus.path }}" + prometheus.io/port: "{{ .Values.global.prometheus.port }}" + {{- end }} + checksum/mongodb-config: {{ include (print $.Template.BasePath "/mongodb-secret.yaml") . | sha256sum }} + {{- with .Values.policies.additionalAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.rbac.enabled }} + serviceAccountName: {{ template "ditto.serviceAccountName" . }} + {{- end }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + fsGroup: 1000 + initContainers: + {{- if .Values.global.logging.logFiles.enabled }} + - name: change-volume-owner + image: busybox + securityContext: + runAsUser: 0 + command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ] + volumeMounts: + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + containers: + - name: {{ .Chart.Name }}-policies + image: {{ printf "%s:%s" .Values.policies.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.policies.image.tag ) ) }} + imagePullPolicy: {{ .Values.policies.image.pullPolicy }} + env: + {{- if not .Values.global.logging.customConfigFile.enabled }} + - name: DITTO_LOGGING_DISABLE_SYSOUT_LOG + value: "{{ if .Values.global.logging.sysout.enabled }}false{{ else }}true{{ end }}" + - name: DITTO_LOGGING_FILE_APPENDER + value: "{{ if .Values.global.logging.logFiles.enabled }}true{{ else }}false{{ end }}" + {{- end }} + - name: DITTO_TRACING_ENABLED + value: "{{ .Values.global.tracing.enabled }}" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "{{ .Values.global.tracing.otelExporterOtlpEndpoint }}" + - name: DITTO_TRACING_SAMPLER + value: "{{ .Values.global.tracing.sampler }}" + - name: DITTO_TRACING_RANDOM_SAMPLER_PROBABILITY + value: "{{ .Values.global.tracing.randomSampler.probability }}" + - name: DITTO_TRACING_ADAPTIVE_SAMPLER_THROUGHPUT + value: "{{ .Values.global.tracing.adaptiveSampler.throughput }}" + {{- if .Values.global.logging.logstash.enabled }} + - name: DITTO_LOGGING_LOGSTASH_SERVER + value: "{{ .Values.global.logging.logstash.endpoint }}" + {{- end }} + - name: POD_LABEL_SELECTOR + value: "app.kubernetes.io/name=%s" + - name: POD_NAMESPACE + value: {{.Release.Namespace}} + - name: INSTANCE_INDEX + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: DISCOVERY_METHOD + value: "kubernetes-api" + - name: TZ + value: "{{ .Values.global.timezone }}" + - name: JAVA_TOOL_OPTIONS + value: > + {{ .Values.global.jvmOptions }} + -XX:ActiveProcessorCount={{ .Values.policies.jvm.activeProcessorCount }} + -XX:MaxRAMPercentage={{ .Values.policies.jvm.heapRamPercentage }} + -XX:InitialRAMPercentage={{ .Values.policies.jvm.heapRamPercentage }} + -XX:MaxGCPauseMillis={{ .Values.policies.jvm.maxGcPauseMillis }} + {{ .Values.policies.additionalJvmOptions }} + {{- .Values.global.akkaOptions }} + {{- if .Values.global.logging.customConfigFile.enabled }} + -Dlogback.configurationFile=/opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{- range $index, $header := .Values.policies.config.persistence.events.historicalHeadersToPersist }} + "{{ printf "%s%d=%s" "-Dditto.policies.policy.event.historical-headers-to-persist." $index $header }}" + {{- end }} + {{- range $grantIdx, $grant := .Values.policies.config.entityCreation.grants }} + "{{ printf "%s%d%s=%s" "-Dditto.entity-creation.grant." $grantIdx ".resource-types.0" "policy" }}" + {{- range $namespaceIdx, $namespace := $grant.namespaces }} + "{{ printf "%s%d%s%d=%s" "-Dditto.entity-creation.grant." $grantIdx ".namespaces." $namespaceIdx $namespace }}" + {{- end }} + {{- range $subjectIdx, $subject := $grant.authSubjects }} + "{{ printf "%s%d%s%d=%s" "-Dditto.entity-creation.grant." $grantIdx ".auth-subjects." $subjectIdx $subject }}" + {{- end }} + {{- end }} + {{- range $revokeIdx, $revoke := .Values.policies.config.entityCreation.revokes }} + "{{ printf "%s%d%s=%s" "-Dditto.entity-creation.revoke." $revokeIdx ".resource-types.0" "policy" }}" + {{- range $namespaceIdx, $namespace := $revoke.namespaces }} + "{{ printf "%s%d%s%d=%s" "-Dditto.entity-creation.revoke." $revokeIdx ".namespaces." $namespaceIdx $namespace }}" + {{- end }} + {{- range $subjectIdx, $subject := $revoke.authSubjects }} + "{{ printf "%s%d%s%d=%s" "-Dditto.entity-creation.revoke." $revokeIdx ".auth-subjects." $subjectIdx $subject }}" + {{- end }} + {{- end }} + {{ join " " .Values.policies.systemProps }} + - name: MONGO_DB_SSL_ENABLED + value: "{{ if .Values.dbconfig.policies.ssl }}true{{ else }}false{{ end }}" + - name: MONGO_DB_URI + valueFrom: + secretKeyRef: + name: {{ .Values.dbconfig.uriSecret | default ( printf "%s-mongodb-secret" ( include "ditto.fullname" . )) }} + key: policies-uri + - name: MONGO_DB_CONNECTION_MIN_POOL_SIZE + value: "{{ .Values.policies.config.mongodb.minPoolSize }}" + - name: MONGO_DB_CONNECTION_POOL_SIZE + value: "{{ .Values.policies.config.mongodb.maxPoolSize }}" + - name: MONGO_DB_CONNECTION_POOL_IDLE_TIME + value: "{{ .Values.policies.config.mongodb.maxPoolIdleTime }}" + {{- if .Values.global.prometheus.enabled }} + - name: PROMETHEUS_PORT + value: "{{ .Values.global.prometheus.port }}" + {{- end }} + - name: CLUSTER_BS_REQUIRED_CONTACTS + value: "{{ .Values.global.cluster.requiredContactPoints }}" + - name: DITTO_DDATA_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.ddata.numberOfShards }}" + - name: DITTO_DDATA_MAX_DELTA_ELEMENTS + value: "{{ .Values.global.cluster.ddata.maxDeltaElements }}" + - name: CLUSTER_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.numberOfShards }}" + - name: CLUSTER_DOWNING_STABLE_AFTER + value: "{{ .Values.global.cluster.downingStableAfter }}" + - name: CLUSTER_DOWNING_DOWN_ALL_WHEN_UNSTABLE + value: "{{ .Values.global.cluster.downAllWhenUnstable }}" + - name: AKKA_PERSISTENCE_MONGO_JOURNAL_WRITE_CONCERN + value: "{{ .Values.policies.config.mongodb.journalWriteConcern }}" + - name: AKKA_PERSISTENCE_MONGO_SNAPS_WRITE_CONCERN + value: "{{ .Values.policies.config.mongodb.snapsWriteConcern }}" + - name: BREAKER_MAXTRIES + value: "{{ .Values.policies.config.mongodb.journalCircuitBreaker.maxTries }}" + - name: BREAKER_TIMEOUT + value: "{{ .Values.policies.config.mongodb.journalCircuitBreaker.timeout }}" + - name: BREAKER_RESET + value: "{{ .Values.policies.config.mongodb.journalCircuitBreaker.reset }}" + - name: SNAPSHOT_BREAKER_MAXTRIES + value: "{{ .Values.policies.config.mongodb.snapsCircuitBreaker.maxTries }}" + - name: SNAPSHOT_BREAKER_TIMEOUT + value: "{{ .Values.policies.config.mongodb.snapsCircuitBreaker.timeout }}" + - name: SNAPSHOT_BREAKER_RESET + value: "{{ .Values.policies.config.mongodb.snapsCircuitBreaker.reset }}" + - name: POLICY_ACTIVITY_CHECK_INTERVAL + value: "{{ .Values.policies.config.persistence.activityCheckInterval }}" + - name: HEALTH_CHECK_METRICS_REPORTER_RESOLUTION + value: "{{ .Values.policies.config.cleanup.metricsReporter.resolution }}" + - name: HEALTH_CHECK_METRICS_REPORTER_HISTORY + value: "{{ .Values.policies.config.cleanup.metricsReporter.history }}" + - name: CLEANUP_ENABLED + value: "{{ .Values.policies.config.cleanup.enabled }}" + - name: CLEANUP_QUIET_PERIOD + value: "{{ .Values.policies.config.cleanup.quietPeriod }}" + - name: CLEANUP_HISTORY_RETENTION_DURATION + value: "{{ .Values.policies.config.cleanup.history.retentionDuration }}" + - name: CLEANUP_INTERVAL + value: "{{ .Values.policies.config.cleanup.interval }}" + - name: CLEANUP_TIMER_THRESHOLD + value: "{{ .Values.policies.config.cleanup.timerThreshold }}" + - name: CLEANUP_CREDITS_PER_BATCH + value: "{{ .Values.policies.config.cleanup.creditsPerBatch }}" + - name: POLICIES_PERSISTENCE_PING_RATE_FREQUENCY + value: "{{ .Values.policies.config.persistence.pingRate.frequency }}" + - name: POLICIES_PERSISTENCE_PING_RATE_ENTITIES + value: "{{ .Values.policies.config.persistence.pingRate.entities }}" + - name: POLICY_SNAPSHOT_INTERVAL + value: "{{ .Values.policies.config.persistence.snapshots.interval }}" + - name: POLICY_SNAPSHOT_THRESHOLD + value: "{{ .Values.policies.config.persistence.snapshots.threshold }}" + {{- if .Values.policies.extraEnv }} + {{- toYaml .Values.policies.extraEnv | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + - name: remoting + containerPort: {{ .Values.akka.remoting.port }} + protocol: TCP + - name: management + containerPort: {{ .Values.akka.mgmthttp.port }} + protocol: TCP + {{- if .Values.global.prometheus.enabled }} + - name: prometheus + protocol: TCP + containerPort: {{ .Values.global.prometheus.port }} + {{- end }} + readinessProbe: + httpGet: + port: management + path: /ready + initialDelaySeconds: {{ .Values.policies.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.policies.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.policies.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.policies.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.policies.readinessProbe.failureThreshold }} + livenessProbe: + httpGet: + port: management + path: /alive + initialDelaySeconds: {{ .Values.policies.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.policies.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.policies.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.policies.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.policies.livenessProbe.failureThreshold }} + volumeMounts: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + mountPath: /opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + subPath: {{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + resources: + requests: + cpu: {{ mulf .Values.policies.resources.cpu 1000 }}m + memory: {{ .Values.policies.resources.memoryMi }}Mi + limits: + # ## no cpu limit to avoid CFS scheduler limits + # ref: https://doc.akka.io/docs/akka/snapshot/additional/deploy.html#in-kubernetes + # cpu: "" + memory: {{ .Values.policies.resources.memoryMi }}Mi + {{- if .Values.openshift.enabled }} + {{- with .Values.openshift.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + {{- end }} + {{- with .Values.policies.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.policies.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.policies.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + configMap: + name: {{ .Release.Name }}-logback-config-policies-xml + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + hostPath: + path: /var/log/ditto + type: DirectoryOrCreate + {{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/policies-pdb.yaml b/deployment/helm/ditto/templates/policies-pdb.yaml new file mode 100644 index 00000000000..e15dd6bd5aa --- /dev/null +++ b/deployment/helm/ditto/templates/policies-pdb.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.policies.podDisruptionBudget.enabled (gt .Values.policies.replicaCount 1.0) -}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "ditto.fullname" . }}-policies + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-policies +{{ include "ditto.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-policies + app.kubernetes.io/instance: {{ .Release.Name }} + minAvailable: {{ .Values.policies.podDisruptionBudget.minAvailable }} +{{- end }} diff --git a/deployment/helm/ditto/templates/policies-podmonitor.yaml b/deployment/helm/ditto/templates/policies-podmonitor.yaml new file mode 100644 index 00000000000..0069a617310 --- /dev/null +++ b/deployment/helm/ditto/templates/policies-podmonitor.yaml @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.policies.podMonitor.enabled .Values.global.prometheus.port -}} +{{- if .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" -}} +--- +kind: PodMonitor +apiVersion: monitoring.coreos.com/v1 +metadata: + name: {{ include "ditto.fullname" . }}-policies + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-policies +{{ include "ditto.labels" . | indent 4 }} +spec: + podMetricsEndpoints: + - targetPort: {{ .Values.global.prometheus.port }} + path: "/" + {{- if .Values.policies.podMonitor.interval }} + interval: {{ .Values.policies.podMonitor.interval }} + {{- end }} + {{- if .Values.policies.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.policies.podMonitor.scrapeTimeout }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-policies + namespaceSelector: + matchNames: + - {{ $.Release.Namespace | quote }} +{{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/role.yaml b/deployment/helm/ditto/templates/role.yaml new file mode 100644 index 00000000000..623ea685b72 --- /dev/null +++ b/deployment/helm/ditto/templates/role.yaml @@ -0,0 +1,24 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.rbac.enabled -}} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ditto.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }} +{{ include "ditto.labels" . | indent 4 }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +{{- end -}} diff --git a/deployment/helm/ditto/templates/rolebinding.yaml b/deployment/helm/ditto/templates/rolebinding.yaml new file mode 100644 index 00000000000..3fdece1eb9f --- /dev/null +++ b/deployment/helm/ditto/templates/rolebinding.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.rbac.enabled -}} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ditto.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }} +{{ include "ditto.labels" . | indent 4 }} +roleRef: + kind: Role + name: {{ include "ditto.fullname" . }} + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: {{ template "ditto.serviceAccountName" . }} +{{- end -}} diff --git a/deployment/helm/ditto/templates/serviceaccount.yaml b/deployment/helm/ditto/templates/serviceaccount.yaml new file mode 100644 index 00000000000..04f60c2146a --- /dev/null +++ b/deployment/helm/ditto/templates/serviceaccount.yaml @@ -0,0 +1,20 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.serviceAccount.create -}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "ditto.serviceAccountName" . }} + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }} +{{ include "ditto.labels" . | indent 4 }} +{{- end -}} diff --git a/deployment/helm/ditto/templates/swaggerui-config.yaml b/deployment/helm/ditto/templates/swaggerui-config.yaml new file mode 100644 index 00000000000..3f035ae4a7a --- /dev/null +++ b/deployment/helm/ditto/templates/swaggerui-config.yaml @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.swaggerui.enabled -}} +{{- $releaseName := .Release.Name -}} +{{- $name := include "ditto.name" . -}} +{{- $labels := include "ditto.labels" . -}} +{{ $root := . }} +{{ range $path, $bytes := .Files.Glob "swaggerui-config/**" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $releaseName }}-{{ $path | replace "/" "-" | replace "." "-" }} + labels: + app.kubernetes.io/name: {{ $name }}-swaggerui-config +{{ $labels | indent 4 }} +data: + {{ $path | replace "swaggerui-config/" ""}}: |- +{{ $root.Files.Get $path | indent 4 }} +--- +{{- end -}} +{{- end -}} diff --git a/deployment/helm/ditto/templates/swaggerui-deployment.yaml b/deployment/helm/ditto/templates/swaggerui-deployment.yaml new file mode 100644 index 00000000000..e6854da2dfa --- /dev/null +++ b/deployment/helm/ditto/templates/swaggerui-deployment.yaml @@ -0,0 +1,114 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.swaggerui.enabled -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ditto.fullname" . }}-swaggerui + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-swaggerui +{{ include "ditto.labels" . | indent 4 }} +spec: + replicas: {{ .Values.swaggerui.replicaCount }} + strategy: + {{- with .Values.swaggerui.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-swaggerui + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-swaggerui + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.swaggerui.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.swaggerui.additionalAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: {{ .Chart.Name }}-swaggerui-init + image: "docker.io/boky/alpine-bootstrap:latest" + imagePullPolicy: {{ .Values.swaggerui.image.pullPolicy }} + command: + - sh + - -ec + - | + mkdir -p /usr/share/nginx/html/openapi + curl -sL https://raw.githubusercontent.com/eclipse/ditto/{{ .Chart.AppVersion }}/documentation/src/main/resources/openapi/ditto-api-2.yml -o /usr/share/nginx/html/openapi/ditto-api-2.yml + sed --in-place "\=- url: 'https://ditto.eclipseprojects.io/api/2'=d" /usr/share/nginx/html/openapi/ditto-api-2.yml + sed --in-place "/description: online Ditto Sandbox/d" /usr/share/nginx/html/openapi/ditto-api-2.yml + echo "removing Google auth from ditto-api-2.yml" + sed --in-place "/- Google:/,+1d" /usr/share/nginx/html/openapi/ditto-api-2.yml + sed --in-place "/ Google:/,+9d" /usr/share/nginx/html/openapi/ditto-api-2.yml + {{- if not .Values.gateway.config.authentication.enablePreAuthentication }} + echo "removing NginxBasic auth from ditto-api-2.yml" + sed --in-place "/- NginxBasic: \[]/d" /usr/share/nginx/html/openapi/ditto-api-2.yml + sed --in-place "/ NginxBasic:/,+3d" /usr/share/nginx/html/openapi/ditto-api-2.yml + {{- end }} + {{- if eq .Values.gateway.config.authentication.devops.authMethod "oauth2" }} + echo "removing DevOpsBasic auth from ditto-api-2.yml" + sed --in-place "/- DevOpsBasic: \[]/d" /usr/share/nginx/html/openapi/ditto-api-2.yml + sed --in-place "/ DevOpsBasic:/,+3d" /usr/share/nginx/html/openapi/ditto-api-2.yml + {{- else }} + echo "removing DevOpsBearer auth from ditto-api-2.yml" + sed --in-place "/- DevOpsBearer: \[]/d" /usr/share/nginx/html/openapi/ditto-api-2.yml + sed --in-place "/ DevOpsBearer:/,+4d" /usr/share/nginx/html/openapi/ditto-api-2.yml + {{- end }} + sed --in-place "s=- url: /api/2=- url: {{ .Values.global.proxyPart }}/api/2=g" /usr/share/nginx/html/openapi/ditto-api-2.yml + cp -rv /usr/share/nginx/html/openapi/. /init-config/ + volumeMounts: + - name: swagger-ui-init-config + mountPath: /init-config + containers: + - name: {{ .Chart.Name }}-swaggerui + image: "{{ .Values.swaggerui.image.repository }}:{{ .Values.swaggerui.image.tag }}" + imagePullPolicy: {{ .Values.swaggerui.image.pullPolicy }} + env: + - name: QUERY_CONFIG_ENABLED + value: "true" + {{- if .Values.swaggerui.extraEnv }} + {{- toYaml .Values.swaggerui.extraEnv | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + resources: + requests: + cpu: {{ mulf .Values.swaggerui.resources.cpu 1000 }}m + memory: {{ .Values.swaggerui.resources.memoryMi }}Mi + limits: + # cpu: "" + memory: {{ .Values.swaggerui.resources.memoryMi }}Mi + volumeMounts: + - name: swagger-ui-init-config + mountPath: /usr/share/nginx/html/openapi + - name: swaggerui-index + mountPath: /usr/share/nginx/html/index.html + subPath: index.html + volumes: + - name: swagger-ui-init-config + emptyDir: {} + - name: swaggerui-index + configMap: + name: {{ .Release.Name }}-swaggerui-config-index-html +{{- end }} diff --git a/deployment/helm/ditto/templates/swaggerui-pdb.yaml b/deployment/helm/ditto/templates/swaggerui-pdb.yaml new file mode 100644 index 00000000000..43abca8abc2 --- /dev/null +++ b/deployment/helm/ditto/templates/swaggerui-pdb.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.swaggerui.podDisruptionBudget.enabled (gt .Values.swaggerui.replicaCount 1.0) -}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "ditto.fullname" . }}-swaggerui + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-swaggerui +{{ include "ditto.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-swaggerui + app.kubernetes.io/instance: {{ .Release.Name }} + minAvailable: {{ .Values.swaggerui.podDisruptionBudget.minAvailable }} +{{- end }} diff --git a/deployment/helm/ditto/templates/swaggerui-service.yaml b/deployment/helm/ditto/templates/swaggerui-service.yaml new file mode 100644 index 00000000000..bb330b57059 --- /dev/null +++ b/deployment/helm/ditto/templates/swaggerui-service.yaml @@ -0,0 +1,33 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.swaggerui.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ditto.fullname" . }}-swaggerui + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-swaggerui +{{ include "ditto.labels" . | indent 4 }} + {{- with .Values.swaggerui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ports: + - port: {{ .Values.swaggerui.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "ditto.name" . }}-swaggerui + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deployment/helm/ditto/templates/things-deployment.yaml b/deployment/helm/ditto/templates/things-deployment.yaml new file mode 100644 index 00000000000..a5355985557 --- /dev/null +++ b/deployment/helm/ditto/templates/things-deployment.yaml @@ -0,0 +1,314 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.things.enabled -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ditto.fullname" . }}-things + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-things +{{ include "ditto.labels" . | indent 4 }} +spec: + replicas: {{ .Values.things.replicaCount }} + strategy: + {{- with .Values.things.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.things.minReadySeconds }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-things + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-things + app.kubernetes.io/instance: {{ .Release.Name }} + actorSystemName: {{ .Values.akka.actorSystemName }} + {{- with .Values.things.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.global.prometheus.enabled }} + prometheus.io/scrape: "true" + prometheus.io/path: "{{ .Values.global.prometheus.path }}" + prometheus.io/port: "{{ .Values.global.prometheus.port }}" + {{- end }} + checksum/mongodb-config: {{ include (print $.Template.BasePath "/mongodb-secret.yaml") . | sha256sum }} + {{- with .Values.things.additionalAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.rbac.enabled }} + serviceAccountName: {{ template "ditto.serviceAccountName" . }} + {{- end }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + fsGroup: 1000 + initContainers: + {{- if .Values.global.logging.logFiles.enabled }} + - name: change-volume-owner + image: busybox + securityContext: + runAsUser: 0 + command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ] + volumeMounts: + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + containers: + - name: {{ .Chart.Name }}-things + image: {{ printf "%s:%s" .Values.things.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.things.image.tag ) ) }} + imagePullPolicy: {{ .Values.things.image.pullPolicy }} + env: + {{- if not .Values.global.logging.customConfigFile.enabled }} + - name: DITTO_LOGGING_DISABLE_SYSOUT_LOG + value: "{{ if .Values.global.logging.sysout.enabled }}false{{ else }}true{{ end }}" + - name: DITTO_LOGGING_FILE_APPENDER + value: "{{ if .Values.global.logging.logFiles.enabled }}true{{ else }}false{{ end }}" + {{- end }} + - name: DITTO_TRACING_ENABLED + value: "{{ .Values.global.tracing.enabled }}" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "{{ .Values.global.tracing.otelExporterOtlpEndpoint }}" + - name: DITTO_TRACING_SAMPLER + value: "{{ .Values.global.tracing.sampler }}" + - name: DITTO_TRACING_RANDOM_SAMPLER_PROBABILITY + value: "{{ .Values.global.tracing.randomSampler.probability }}" + - name: DITTO_TRACING_ADAPTIVE_SAMPLER_THROUGHPUT + value: "{{ .Values.global.tracing.adaptiveSampler.throughput }}" + {{- if .Values.global.logging.logstash.enabled }} + - name: DITTO_LOGGING_LOGSTASH_SERVER + value: "{{ .Values.global.logging.logstash.endpoint }}" + {{- end }} + - name: POD_LABEL_SELECTOR + value: "app.kubernetes.io/name=%s" + - name: POD_NAMESPACE + value: {{.Release.Namespace}} + - name: INSTANCE_INDEX + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: DISCOVERY_METHOD + value: "kubernetes-api" + - name: TZ + value: "{{ .Values.global.timezone }}" + - name: JAVA_TOOL_OPTIONS + value: > + {{ .Values.global.jvmOptions }} + -XX:ActiveProcessorCount={{ .Values.things.jvm.activeProcessorCount }} + -XX:MaxRAMPercentage={{ .Values.things.jvm.heapRamPercentage }} + -XX:InitialRAMPercentage={{ .Values.things.jvm.heapRamPercentage }} + -XX:MaxGCPauseMillis={{ .Values.things.jvm.maxGcPauseMillis }} + {{ .Values.things.additionalJvmOptions }} + {{- .Values.global.akkaOptions }} + {{- if .Values.global.logging.customConfigFile.enabled }} + -Dlogback.configurationFile=/opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{- range $index, $header := .Values.things.config.persistence.events.historicalHeadersToPersist }} + "{{ printf "%s%d=%s" "-Dditto.things.thing.event.historical-headers-to-persist." $index $header }}" + {{- end }} + {{- range $grantIdx, $grant := .Values.things.config.entityCreation.grants }} + "{{ printf "%s%d%s=%s" "-Dditto.entity-creation.grant." $grantIdx ".resource-types.0" "thing" }}" + {{- range $namespaceIdx, $namespace := $grant.namespaces }} + "{{ printf "%s%d%s%d=%s" "-Dditto.entity-creation.grant." $grantIdx ".namespaces." $namespaceIdx $namespace }}" + {{- end }} + {{- range $subjectIdx, $subject := $grant.authSubjects }} + "{{ printf "%s%d%s%d=%s" "-Dditto.entity-creation.grant." $grantIdx ".auth-subjects." $subjectIdx $subject }}" + {{- end }} + {{- end }} + {{- range $revokeIdx, $revoke := .Values.things.config.entityCreation.revokes }} + "{{ printf "%s%d%s=%s" "-Dditto.entity-creation.revoke." $revokeIdx ".resource-types.0" "thing" }}" + {{- range $namespaceIdx, $namespace := $revoke.namespaces }} + "{{ printf "%s%d%s%d=%s" "-Dditto.entity-creation.revoke." $revokeIdx ".namespaces." $namespaceIdx $namespace }}" + {{- end }} + {{- range $subjectIdx, $subject := $revoke.authSubjects }} + "{{ printf "%s%d%s%d=%s" "-Dditto.entity-creation.revoke." $revokeIdx ".auth-subjects." $subjectIdx $subject }}" + {{- end }} + {{- end }} + '-Dditto.things.wot.to-thing-description.json-template={{ .Values.things.config.wot.tdJsonTemplate | replace "\n" "" | replace "\\\"" "\"" }}' + {{ join " " .Values.things.systemProps }} + - name: MONGO_DB_SSL_ENABLED + value: "{{ if .Values.dbconfig.things.ssl }}true{{ else }}false{{ end }}" + - name: MONGO_DB_URI + valueFrom: + secretKeyRef: + name: {{ .Values.dbconfig.uriSecret | default ( printf "%s-mongodb-secret" ( include "ditto.fullname" . )) }} + key: things-uri + - name: MONGO_DB_CONNECTION_MIN_POOL_SIZE + value: "{{ .Values.things.config.mongodb.minPoolSize }}" + - name: MONGO_DB_CONNECTION_POOL_SIZE + value: "{{ .Values.things.config.mongodb.maxPoolSize }}" + - name: MONGO_DB_CONNECTION_POOL_IDLE_TIME + value: "{{ .Values.things.config.mongodb.maxPoolIdleTime }}" + {{- if .Values.global.prometheus.enabled }} + - name: PROMETHEUS_PORT + value: "{{ .Values.global.prometheus.port }}" + {{- end }} + - name: CLUSTER_BS_REQUIRED_CONTACTS + value: "{{ .Values.global.cluster.requiredContactPoints }}" + - name: DITTO_DDATA_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.ddata.numberOfShards }}" + - name: DITTO_DDATA_MAX_DELTA_ELEMENTS + value: "{{ .Values.global.cluster.ddata.maxDeltaElements }}" + - name: CLUSTER_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.numberOfShards }}" + - name: CLUSTER_DOWNING_STABLE_AFTER + value: "{{ .Values.global.cluster.downingStableAfter }}" + - name: CLUSTER_DOWNING_DOWN_ALL_WHEN_UNSTABLE + value: "{{ .Values.global.cluster.downAllWhenUnstable }}" + - name: AKKA_PERSISTENCE_MONGO_JOURNAL_WRITE_CONCERN + value: "{{ .Values.things.config.mongodb.journalWriteConcern }}" + - name: AKKA_PERSISTENCE_MONGO_SNAPS_WRITE_CONCERN + value: "{{ .Values.things.config.mongodb.snapsWriteConcern }}" + - name: BREAKER_MAXTRIES + value: "{{ .Values.things.config.mongodb.journalCircuitBreaker.maxTries }}" + - name: BREAKER_TIMEOUT + value: "{{ .Values.things.config.mongodb.journalCircuitBreaker.timeout }}" + - name: BREAKER_RESET + value: "{{ .Values.things.config.mongodb.journalCircuitBreaker.reset }}" + - name: SNAPSHOT_BREAKER_MAXTRIES + value: "{{ .Values.things.config.mongodb.snapsCircuitBreaker.maxTries }}" + - name: SNAPSHOT_BREAKER_TIMEOUT + value: "{{ .Values.things.config.mongodb.snapsCircuitBreaker.timeout }}" + - name: SNAPSHOT_BREAKER_RESET + value: "{{ .Values.things.config.mongodb.snapsCircuitBreaker.reset }}" + - name: THING_ACTIVITY_CHECK_INTERVAL + value: "{{ .Values.things.config.persistence.activityCheckInterval }}" + - name: HEALTH_CHECK_METRICS_REPORTER_RESOLUTION + value: "{{ .Values.things.config.cleanup.metricsReporter.resolution }}" + - name: HEALTH_CHECK_METRICS_REPORTER_HISTORY + value: "{{ .Values.things.config.cleanup.metricsReporter.history }}" + - name: CLEANUP_ENABLED + value: "{{ .Values.things.config.cleanup.enabled }}" + - name: CLEANUP_QUIET_PERIOD + value: "{{ .Values.things.config.cleanup.quietPeriod }}" + - name: CLEANUP_HISTORY_RETENTION_DURATION + value: "{{ .Values.things.config.cleanup.history.retentionDuration }}" + - name: CLEANUP_INTERVAL + value: "{{ .Values.things.config.cleanup.interval }}" + - name: CLEANUP_TIMER_THRESHOLD + value: "{{ .Values.things.config.cleanup.timerThreshold }}" + - name: CLEANUP_CREDITS_PER_BATCH + value: "{{ .Values.things.config.cleanup.creditsPerBatch }}" + - name: THING_SNAPSHOT_INTERVAL + value: "{{ .Values.things.config.persistence.snapshots.interval }}" + - name: THING_SNAPSHOT_THRESHOLD + value: "{{ .Values.things.config.persistence.snapshots.threshold }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_ENABLED + value: "{{ .Values.things.config.policiesEnforcer.cache.enabled }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_MAX_SIZE + value: "{{ .Values.things.config.policiesEnforcer.cache.maxSize }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_EXPIRE_AFTER_WRITE + value: "{{ .Values.things.config.policiesEnforcer.cache.expireAfterWrite }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_EXPIRE_AFTER_ACCESS + value: "{{ .Values.things.config.policiesEnforcer.cache.expireAfterAccess }}" + - name: THINGS_WOT_TO_THING_DESCRIPTION_BASE_PREFIX + value: "{{ .Values.things.config.wot.tdBasePrefix }}" + {{- if .Values.things.extraEnv }} + {{- toYaml .Values.things.extraEnv | nindent 12 }} + {{- end }} + ports: + - name: remoting + containerPort: {{ .Values.akka.remoting.port }} + protocol: TCP + - name: management + containerPort: {{ .Values.akka.mgmthttp.port }} + protocol: TCP + {{- if .Values.global.prometheus.enabled }} + - name: prometheus + protocol: TCP + containerPort: {{ .Values.global.prometheus.port }} + {{- end }} + readinessProbe: + httpGet: + port: management + path: /ready + initialDelaySeconds: {{ .Values.things.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.things.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.things.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.things.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.things.readinessProbe.failureThreshold }} + livenessProbe: + httpGet: + port: management + path: /alive + initialDelaySeconds: {{ .Values.things.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.things.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.things.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.things.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.things.livenessProbe.failureThreshold }} + volumeMounts: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + mountPath: /opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + subPath: {{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + resources: + requests: + cpu: {{ mulf .Values.things.resources.cpu 1000 }}m + memory: {{ .Values.things.resources.memoryMi }}Mi + limits: + # ## no cpu limit to avoid CFS scheduler limits + # ref: https://doc.akka.io/docs/akka/snapshot/additional/deploy.html#in-kubernetes + # cpu: "" + memory: {{ .Values.things.resources.memoryMi }}Mi + {{- if .Values.openshift.enabled }} + {{- with .Values.openshift.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + {{- end }} + {{- with .Values.things.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.things.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.things.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + configMap: + name: {{ .Release.Name }}-logback-config-things-xml + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + hostPath: + path: /var/log/ditto + type: DirectoryOrCreate + {{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/things-pdb.yaml b/deployment/helm/ditto/templates/things-pdb.yaml new file mode 100644 index 00000000000..8f73693f60d --- /dev/null +++ b/deployment/helm/ditto/templates/things-pdb.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.things.podDisruptionBudget.enabled (gt .Values.things.replicaCount 1.0) -}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "ditto.fullname" . }}-things + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-things +{{ include "ditto.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-things + app.kubernetes.io/instance: {{ .Release.Name }} + minAvailable: {{ .Values.things.podDisruptionBudget.minAvailable }} +{{- end }} diff --git a/deployment/helm/ditto/templates/things-podmonitor.yaml b/deployment/helm/ditto/templates/things-podmonitor.yaml new file mode 100644 index 00000000000..e9816d72801 --- /dev/null +++ b/deployment/helm/ditto/templates/things-podmonitor.yaml @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.things.podMonitor.enabled .Values.global.prometheus.port -}} +{{- if .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" -}} +--- +kind: PodMonitor +apiVersion: monitoring.coreos.com/v1 +metadata: + name: {{ include "ditto.fullname" . }}-things + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-things +{{ include "ditto.labels" . | indent 4 }} +spec: + podMetricsEndpoints: + - targetPort: {{ .Values.global.prometheus.port }} + path: "/" + {{- if .Values.things.podMonitor.interval }} + interval: {{ .Values.things.podMonitor.interval }} + {{- end }} + {{- if .Values.things.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.things.podMonitor.scrapeTimeout }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-things + namespaceSelector: + matchNames: + - {{ $.Release.Namespace | quote }} +{{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/thingssearch-deployment.yaml b/deployment/helm/ditto/templates/thingssearch-deployment.yaml new file mode 100644 index 00000000000..7c32d457ebf --- /dev/null +++ b/deployment/helm/ditto/templates/thingssearch-deployment.yaml @@ -0,0 +1,294 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if .Values.thingsSearch.enabled -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ditto.fullname" . }}-thingssearch + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-thingssearch +{{ include "ditto.labels" . | indent 4 }} +spec: + replicas: {{ .Values.thingsSearch.replicaCount }} + strategy: + {{- with .Values.thingsSearch.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.thingsSearch.minReadySeconds }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-thingssearch + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-thingssearch + app.kubernetes.io/instance: {{ .Release.Name }} + actorSystemName: {{ .Values.akka.actorSystemName }} + {{- with .Values.thingsSearch.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.global.prometheus.enabled }} + prometheus.io/scrape: "true" + prometheus.io/path: "{{ .Values.global.prometheus.path }}" + prometheus.io/port: "{{ .Values.global.prometheus.port }}" + {{- end }} + checksum/mongodb-config: {{ include (print $.Template.BasePath "/mongodb-secret.yaml") . | sha256sum }} + {{- with .Values.thingsSearch.additionalAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.rbac.enabled }} + serviceAccountName: {{ template "ditto.serviceAccountName" . }} + {{- end }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + fsGroup: 1000 + initContainers: + {{- if .Values.global.logging.logFiles.enabled }} + - name: change-volume-owner + image: busybox + securityContext: + runAsUser: 0 + command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ] + volumeMounts: + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + containers: + - name: {{ .Chart.Name }}-thingssearch + image: {{ printf "%s:%s" .Values.thingsSearch.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.thingsSearch.image.tag ) ) }} + imagePullPolicy: {{ .Values.thingsSearch.image.pullPolicy }} + env: + {{- if not .Values.global.logging.customConfigFile.enabled }} + - name: DITTO_LOGGING_DISABLE_SYSOUT_LOG + value: "{{ if .Values.global.logging.sysout.enabled }}false{{ else }}true{{ end }}" + - name: DITTO_LOGGING_FILE_APPENDER + value: "{{ if .Values.global.logging.logFiles.enabled }}true{{ else }}false{{ end }}" + {{- end }} + - name: DITTO_TRACING_ENABLED + value: "{{ .Values.global.tracing.enabled }}" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "{{ .Values.global.tracing.otelExporterOtlpEndpoint }}" + - name: DITTO_TRACING_SAMPLER + value: "{{ .Values.global.tracing.sampler }}" + - name: DITTO_TRACING_RANDOM_SAMPLER_PROBABILITY + value: "{{ .Values.global.tracing.randomSampler.probability }}" + - name: DITTO_TRACING_ADAPTIVE_SAMPLER_THROUGHPUT + value: "{{ .Values.global.tracing.adaptiveSampler.throughput }}" + {{- if .Values.global.logging.logstash.enabled }} + - name: DITTO_LOGGING_LOGSTASH_SERVER + value: "{{ .Values.global.logging.logstash.endpoint }}" + {{- end }} + - name: POD_LABEL_SELECTOR + value: "app.kubernetes.io/name=%s" + - name: POD_NAMESPACE + value: {{.Release.Namespace}} + - name: INSTANCE_INDEX + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: DISCOVERY_METHOD + value: "kubernetes-api" + - name: TZ + value: "{{ .Values.global.timezone }}" + - name: JAVA_TOOL_OPTIONS + value: > + {{ .Values.global.jvmOptions }} + -XX:ActiveProcessorCount={{ .Values.thingsSearch.jvm.activeProcessorCount }} + -XX:MaxRAMPercentage={{ .Values.thingsSearch.jvm.heapRamPercentage }} + -XX:InitialRAMPercentage={{ .Values.thingsSearch.jvm.heapRamPercentage }} + -XX:MaxGCPauseMillis={{ .Values.thingsSearch.jvm.maxGcPauseMillis }} + {{ .Values.thingsSearch.additionalJvmOptions }} + {{- .Values.global.akkaOptions }} + {{- if .Values.global.logging.customConfigFile.enabled }} + -Dlogback.configurationFile=/opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{ join " " .Values.thingsSearch.systemProps }} + - name: MONGO_DB_SSL_ENABLED + value: "{{ if .Values.dbconfig.thingsSearch.ssl }}true{{ else }}false{{ end }}" + - name: MONGO_DB_URI + valueFrom: + secretKeyRef: + name: {{ .Values.dbconfig.uriSecret | default ( printf "%s-mongodb-secret" ( include "ditto.fullname" . )) }} + key: thingsSearch-uri + - name: MONGO_DB_CONNECTION_MIN_POOL_SIZE + value: "{{ .Values.thingsSearch.config.mongodb.minPoolSize }}" + - name: MONGO_DB_CONNECTION_POOL_SIZE + value: "{{ .Values.thingsSearch.config.mongodb.maxPoolSize }}" + - name: MONGO_DB_CONNECTION_POOL_IDLE_TIME + value: "{{ .Values.thingsSearch.config.mongodb.maxPoolIdleTime }}" + {{- if .Values.global.prometheus.enabled }} + - name: PROMETHEUS_PORT + value: "{{ .Values.global.prometheus.port }}" + {{- end }} + - name: CLUSTER_BS_REQUIRED_CONTACTS + value: "{{ .Values.global.cluster.requiredContactPoints }}" + - name: DITTO_DDATA_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.ddata.numberOfShards }}" + - name: DITTO_DDATA_MAX_DELTA_ELEMENTS + value: "{{ .Values.global.cluster.ddata.maxDeltaElements }}" + - name: CLUSTER_NUMBER_OF_SHARDS + value: "{{ .Values.global.cluster.numberOfShards }}" + - name: CLUSTER_DOWNING_STABLE_AFTER + value: "{{ .Values.global.cluster.downingStableAfter }}" + - name: CLUSTER_DOWNING_DOWN_ALL_WHEN_UNSTABLE + value: "{{ .Values.global.cluster.downAllWhenUnstable }}" + - name: MONGO_DB_READ_PREFERENCE + value: "{{ .Values.thingsSearch.config.mongodb.searchReadPreference }}" + - name: QUERY_PERSISTENCE_MONGO_DB_READ_CONCERN + value: "{{ .Values.thingsSearch.config.mongodb.queryReadConcern }}" + - name: MONGO_DB_WRITE_CONCERN + value: "{{ .Values.thingsSearch.config.mongodb.searchWriteConcern }}" + - name: UPDATER_PERSISTENCE_MONGO_DB_READ_CONCERN + value: "{{ .Values.thingsSearch.config.mongodb.updaterPersistenceReadConcern }}" + - name: UPDATER_PERSISTENCE_MONGO_DB_READ_PREFERENCE + value: "{{ .Values.thingsSearch.config.mongodb.updaterPersistenceReadPreference }}" + - name: THINGS_SEARCH_UPDATER_STREAM_PERSISTENCE_WITH_ACKS_WRITE_CONCERN + value: "{{ .Values.thingsSearch.config.mongodb.searchWithAcksWriteConcern }}" + - name: THINGS_SEARCH_UPDATER_STREAM_POLICY_CACHE_SIZE + value: "{{ .Values.thingsSearch.config.updater.stream.policiesEnforcer.cache.maxSize }}" + - name: THINGS_SEARCH_UPDATER_STREAM_POLICY_CACHE_EXPIRY + value: "{{ .Values.thingsSearch.config.updater.stream.policiesEnforcer.cache.expireAfterWrite }}" + - name: THINGS_SEARCH_UPDATER_STREAM_POLICY_CACHE_EXPIRY_AFTER_ACCESS + value: "{{ .Values.thingsSearch.config.updater.stream.policiesEnforcer.cache.expireAfterAccess }}" + - name: THINGS_SEARCH_UPDATER_STREAM_THING_CACHE_SIZE + value: "{{ .Values.thingsSearch.config.updater.stream.thingCache.maxSize }}" + - name: THINGS_SEARCH_UPDATER_STREAM_THING_CACHE_EXPIRY + value: "{{ .Values.thingsSearch.config.updater.stream.thingCache.expireAfterWrite }}" + - name: THINGS_SEARCH_UPDATER_STREAM_THING_CACHE_EXPIRY_AFTER_ACCESS + value: "{{ .Values.thingsSearch.config.updater.stream.thingCache.expireAfterAccess }}" + - name: THINGS_SEARCH_UPDATER_STREAM_RETRIEVAL_PARALLELISM + value: "{{ .Values.thingsSearch.config.updater.stream.retrievalParallelism }}" + - name: THINGS_SEARCH_UPDATER_STREAM_PERSISTENCE_PARALLELISM + value: "{{ .Values.thingsSearch.config.updater.stream.persistence.parallelism }}" + - name: ACTIVITY_CHECK_INTERVAL + value: "{{ .Values.thingsSearch.config.updater.activityCheckInterval }}" + - name: BACKGROUND_SYNC_ENABLED + value: "{{ .Values.thingsSearch.config.updater.backgroundSync.enabled }}" + - name: BACKGROUND_SYNC_QUIET_PERIOD + value: "{{ .Values.thingsSearch.config.updater.backgroundSync.quietPeriod }}" + - name: BACKGROUND_SYNC_IDLE_TIMEOUT + value: "{{ .Values.thingsSearch.config.updater.backgroundSync.idleTimeout }}" + - name: BACKGROUND_SYNC_TOLERANCE_WINDOW + value: "{{ .Values.thingsSearch.config.updater.backgroundSync.toleranceWindow }}" + - name: BACKGROUND_SYNC_KEEP_EVENTS + value: "{{ .Values.thingsSearch.config.updater.backgroundSync.keepEvents }}" + - name: BACKGROUND_SYNC_THROTTLE_THROUGHPUT + value: "{{ .Values.thingsSearch.config.updater.backgroundSync.throttle.throughput }}" + - name: BACKGROUND_SYCN_THROTTLE_PERIOD + value: "{{ .Values.thingsSearch.config.updater.backgroundSync.throttle.period }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_MAX_SIZE + value: "{{ .Values.thingsSearch.config.updater.stream.policiesEnforcer.cache.maxSize }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_EXPIRE_AFTER_WRITE + value: "{{ .Values.thingsSearch.config.updater.stream.policiesEnforcer.cache.expireAfterWrite }}" + - name: DITTO_POLICIES_ENFORCER_CACHE_EXPIRE_AFTER_ACCESS + value: "{{ .Values.thingsSearch.config.updater.stream.policiesEnforcer.cache.expireAfterAccess }}" + {{- if .Values.thingsSearch.extraEnv }} + {{- toYaml .Values.thingsSearch.extraEnv | nindent 12 }} + {{- end }} + ports: + - name: remoting + containerPort: {{ .Values.akka.remoting.port }} + protocol: TCP + - name: management + containerPort: {{ .Values.akka.mgmthttp.port }} + protocol: TCP + {{- if .Values.global.prometheus.enabled }} + - name: prometheus + protocol: TCP + containerPort: {{ .Values.global.prometheus.port }} + {{- end }} + readinessProbe: + httpGet: + port: management + path: /ready + initialDelaySeconds: {{ .Values.thingsSearch.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.thingsSearch.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.thingsSearch.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.thingsSearch.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.thingsSearch.readinessProbe.failureThreshold }} + livenessProbe: + httpGet: + port: management + path: /alive + initialDelaySeconds: {{ .Values.thingsSearch.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.thingsSearch.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.thingsSearch.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.thingsSearch.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.thingsSearch.livenessProbe.failureThreshold }} + volumeMounts: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + mountPath: /opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }} + subPath: {{ .Values.global.logging.customConfigFile.fileName }} + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + mountPath: /var/log/ditto + {{- end }} + resources: + requests: + cpu: {{ mulf .Values.thingsSearch.resources.cpu 1000 }}m + memory: {{ .Values.thingsSearch.resources.memoryMi }}Mi + limits: + # ## no cpu limit to avoid CFS scheduler limits + # ref: https://doc.akka.io/docs/akka/snapshot/additional/deploy.html#in-kubernetes + # cpu: "" + memory: {{ .Values.thingsSearch.resources.memoryMi }}Mi + {{- if .Values.openshift.enabled }} + {{- with .Values.openshift.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + {{- end }} + {{- with .Values.thingsSearch.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.thingsSearch.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.thingsSearch.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.global.logging.customConfigFile.enabled }} + - name: ditto-custom-log-config + configMap: + name: {{ .Release.Name }}-logback-config-thingssearch-xml + {{- end }} + {{- if .Values.global.logging.logFiles.enabled }} + - name: ditto-log-files-directory + hostPath: + path: /var/log/ditto + type: DirectoryOrCreate + {{- end }} +{{- end }} diff --git a/deployment/helm/ditto/templates/thingssearch-pdb.yaml b/deployment/helm/ditto/templates/thingssearch-pdb.yaml new file mode 100644 index 00000000000..c7651415a06 --- /dev/null +++ b/deployment/helm/ditto/templates/thingssearch-pdb.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.things.podDisruptionBudget.enabled (gt .Values.things.replicaCount 1.0) -}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "ditto.fullname" . }}-thingssearch + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-thingssearch +{{ include "ditto.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-thingssearch + app.kubernetes.io/instance: {{ .Release.Name }} + minAvailable: {{ .Values.things.podDisruptionBudget.minAvailable }} +{{- end }} diff --git a/deployment/helm/ditto/templates/thingssearch-podmonitor.yaml b/deployment/helm/ditto/templates/thingssearch-podmonitor.yaml new file mode 100644 index 00000000000..7f83efb6a58 --- /dev/null +++ b/deployment/helm/ditto/templates/thingssearch-podmonitor.yaml @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +{{- if and .Values.thingsSearch.podMonitor.enabled .Values.global.prometheus.port -}} +{{- if .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" -}} +--- +kind: PodMonitor +apiVersion: monitoring.coreos.com/v1 +metadata: + name: {{ include "ditto.fullname" . }}-thingssearch + labels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-thingssearch +{{ include "ditto.labels" . | indent 4 }} +spec: + podMetricsEndpoints: + - targetPort: {{ .Values.global.prometheus.port }} + path: "/" + {{- if .Values.thingsSearch.podMonitor.interval }} + interval: {{ .Values.thingsSearch.podMonitor.interval }} + {{- end }} + {{- if .Values.thingsSearch.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.thingsSearch.podMonitor.scrapeTimeout }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-thingssearch + namespaceSelector: + matchNames: + - {{ $.Release.Namespace | quote }} +{{- end }} +{{- end }} diff --git a/deployment/helm/ditto/values.yaml b/deployment/helm/ditto/values.yaml new file mode 100644 index 00000000000..2f6649605e2 --- /dev/null +++ b/deployment/helm/ditto/values.yaml @@ -0,0 +1,1586 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +--- +# Default values for ditto. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +serviceAccount: + # create controls whether a service account should be created + create: true + # name is the name of the service account to use + # If not set and create is true, a name is generated using the fullname template + name: + +rbac: + # enabled controls whether RBAC resources will be created + enabled: true + +nameOverride: "" +fullnameOverride: "" + +## ---------------------------------------------------------------------------- +## global configuration shared by all components +global: + # cluster holds the configuration for the Ditto/Akka cluster + cluster: + # requiredContactPoints defines the total amount of replicas in the Ditto cluster + # only if this amount is "seen" during cluster formation, the cluster can form itself + requiredContactPoints: 5 + # ddata holds the "Distributed Data" configuration: + ddata: + # numberOfShards defines whether ddata structures should be shared (if >1) + # this is needed in case a lot of event subscribers (thousands) are connected simultaneously + numberOfShards: 1 + # maxDeltaElements defines how many elements should be synced with a single cluster message + # if numberOfShards is > 1, it makes sense to keep maxDeltaElements lower + # so that the message size for remoting is not exceeding the configured max message size + maxDeltaElements: 1 + # numberOfShards configures the sharding applied for things/policies/connections based on their ID + # as a rule of thumb: should be factor ten of the amount of cluster replicas for an entity + numberOfShards: 50 + # downingStableAfter is a configuration of the Akka SBR (split brain resolver) + # how to find the right value: https://doc.akka.io/docs/akka/current/split-brain-resolver.html + downingStableAfter: 15s + # downAllWhenUnstable is a configuration of the Akka SBR (split brain resolver) + downAllWhenUnstable: "on" + # basicAuthUsers configures (as a map) several user/password combinations which the nginx of the Ditto chart will authenticate + basicAuthUsers: {} + # ditto: + # user: ditto + # password: ditto + # hashedBasicAuthUsers configures a list of hashed .htpasswd username/password entries + hashedBasicAuthUsers: [] + # jwtOnly controls whether only OpenID-Connect authentication is supported + # if false, both OpenID-Connect and basicAuth via nginx (see above "basicAuthUsers" and "hashedBasicAuthUsers") is used + # ref: https://www.eclipse.dev/ditto/installation-operating.html#openid-connect + jwtOnly: false + # jvmOptions defines the JVM options applied to all Ditto services running in the JVM, it is put in JAVA_TOOL_OPTIONS + jvmOptions: > + -XX:+ExitOnOutOfMemoryError + -XX:+UseContainerSupport + -XX:+UseStringDeduplication + -Xss512k + -XX:MaxMetaspaceSize=256m + -XX:+UseG1GC + -Djava.net.preferIPv4Stack=true + akkaOptions: > + -Dakka.management.cluster.bootstrap.contact-point-discovery.port-name=management + -Dakka.cluster.failure-detector.threshold=15.0 + -Dakka.cluster.failure-detector.expected-response-after=3s + -Dakka.cluster.failure-detector.acceptable-heartbeat-pause=7s + -Dakka.persistence.journal-plugin-fallback.recovery-event-timeout=30s + -Dakka.persistence.max-concurrent-recoveries=100 + -Dakka.cluster.sharding.updating-state-timeout=20s + -Dakka.cluster.shutdown-after-unsuccessful-join-seed-nodes=120s + # timezone defines the timezone to configure the JVM with + timezone: Europe/Berlin + # imagePullSecrets will be added to every deployment + imagePullSecrets: [] + # proxyPart configures a reverse proxy part to be added in front of the Ditto API endpoints: + proxyPart: "" + # prometheus holds the Prometheus specific configuration + prometheus: + # enabled controls whether scrape config annotation will be added to pod templates + enabled: true + # path where prometheus metric will be provided + path: "/" + # port where prometheus metrics will be provided + port: 9095 + # logging the logging configuration for Ditto + logging: + # sysout holds the logging to SYSOUT config + sysout: + # enabled defines whether to log to SYSOUT + enabled: true + # logstash configures if logs should be pushed to a logstash endpoint + logstash: + # enabled defines whether to log to logstash + enabled: false + # endpoint configures the logstash endpoint to send logs to + endpoint: "" + # logFiles defines logging to log files config + logFiles: + # enabled whether to write logs to log files + # log files can be found on the host under /var/log/ditto + enabled: false + # customConfigFile configures that a custom "Logback" config file should be used instead of the one bundled + # with Ditto on the classpath + customConfigFile: + # enabled if enabled, a custom logback.xml file added to the Ditto containers will be used for logging configuration + enabled: true + # fileName passed as Java system property "-Dlogback.configurationFile" + fileName: logback.xml + # tracing configuration for Ditto + tracing: + # enabled whether tracing (via OpenTelemetry) is enabled + enabled: false + # otelExporterOtlpEndpoint the OTLP endpoint to report traces to + otelExporterOtlpEndpoint: "http://localhost:4317" + # sampler the tracing sampler to use + # can be one of: + # - always: report all traces. + # - never: don't report any trace. + # - random: randomly decide using the probability defined in the random-sampler.probability setting. + # - adaptive: keeps dynamic samplers for each operation while trying to achieve a set throughput goal. + sampler: never + # randomSampler configures the 'random' sampler + randomSampler: + # probability configures the probability of a span being sampled, must be a value between 0 and 1 + probability: 0.01 + # adaptiveSampler configures the 'adaptive' sampler + adaptiveSampler: + # throughput the throughput goal trying to achieve with the adaptive sampler + throughput: 600 + +## ---------------------------------------------------------------------------- +## dbconfig for mongodb connections +## will be handled as k8s secret as connection uri might contain auth credentials +dbconfig: + # policies the MongoDB configuration for Ditto "policies" service + policies: + uri: mongodb://#{PLACEHOLDER_MONGODB_HOSTNAME}#:27017/ditto + ssl: false + # things the MongoDB configuration for Ditto "things" service + things: + uri: mongodb://#{PLACEHOLDER_MONGODB_HOSTNAME}#:27017/ditto + ssl: false + # connectivity the MongoDB configuration for Ditto "connectivity" service + connectivity: + uri: mongodb://#{PLACEHOLDER_MONGODB_HOSTNAME}#:27017/ditto + ssl: false + # thingsSearch the MongoDB configuration for Ditto "things-search" service + thingsSearch: + uri: mongodb://#{PLACEHOLDER_MONGODB_HOSTNAME}#:27017/ditto + ssl: false + ## If following property is set, an existing secret will be used to retrieve the mongodb connectionUris from. + # uriSecret: my-uri-secret + +## ---------------------------------------------------------------------------- +## ingress configures the Ingress +ingress: + # enabled whether Ingress should be enabled as alternative to the contained nginx + enabled: false + # className is the 'ingressClassName' to configure in the Ingress spec + className: nginx + # host the hostname of the Ingress shared for all: api, ws and ui + host: localhost + # defaultBackendSuffix the suffix to add to the internal fullname to use as Ingress "defaultBackend" + defaultBackendSuffix: nginx + # annotations common annotations for all 3 Ingresses of Ditto + controller: + # enabled whether Ingress controller should be enabled + enabled: false + # namespace for ingress controller, managed by helm, should not be created manually + namespace: ingress-nginx + # Ingress-NGINX version. Check Supported Versions table from https://github.com/kubernetes/ingress-nginx to match k8s version. + nginxIngressVersion: "v1.8.0" + # Nginx Version. Check Supported Versions table from https://github.com/kubernetes/ingress-nginx to match k8s version. + nginxVersion: "1.21.6" + annotations: + nginx.ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/server-snippet: | + charset utf-8; + default_type application/json; + chunked_transfer_encoding off; + + send_timeout 70; # seconds, default: 60 + client_header_buffer_size 8k; # allow longer URIs + headers (default: 1k) + large_client_header_buffers 4 16k; + + proxy_http_version 1.1; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Host $host; + # api the /api, /devops, /status, /overall and /health Ingress configuration + api: + # paths configures ingress paths + paths: + - path: /api + backendSuffix: gateway + - path: /devops + backendSuffix: gateway + - path: /status + backendSuffix: gateway + - path: /stats + backendSuffix: gateway + - path: /overall + backendSuffix: gateway + - path: /health + backendSuffix: gateway + # annotations defines k8s annotations to add to the Ingress + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: "10" + nginx.ingress.kubernetes.io/proxy-send-timeout: "70" + nginx.ingress.kubernetes.io/proxy-read-timeout: "70" + nginx.ingress.kubernetes.io/proxy-next-upstream: "error timeout http_502" + nginx.ingress.kubernetes.io/proxy-next-upstream-tries: "4" + nginx.ingress.kubernetes.io/proxy-next-upstream-timeout: "50" + nginx.ingress.kubernetes.io/proxy-buffer-size: "16k" + nginx.ingress.kubernetes.io/configuration-snippet: | + set $cors '1'; + + if ($request_method = 'OPTIONS') { + set $cors "${cors}o"; + } + + if ($cors = '1') { + add_header 'Access-Control-Allow-Origin' '$http_origin' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, PATCH, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Headers' '$http_access_control_request_headers' always; + add_header 'Access-Control-Expose-Headers' '*' always; + } + + if ($cors = '1o') { + # Tell client that this pre-flight info is valid for 20 days + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Access-Control-Allow-Origin' '$http_origin' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, PATCH, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Headers' '$http_access_control_request_headers' always; + add_header 'Access-Control-Expose-Headers' '*' always; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 200; + } + + if ($request_method = 'OPTIONS') { + add_header 'Content-Type' 'text/plain charset=UTF-8'; + return 405 "Origin $http_origin is not in CORS allow-list, contact your admin to get it added"; + } + + # security relevant headers: + add_header "Content-Security-Policy" "default-src 'none'; frame-ancestors 'none'" always; + add_header "Strict-Transport-Security" "max-age=63072000; includeSubdomains;" always; + add_header "Cache-Control" "no-cache" always; + add_header "X-Content-Type-Options" "nosniff" always; + add_header "X-Frame-Options" "SAMEORIGIN" always; + add_header "X-XSS-Protection" "1; mode=block" always; + # ws the /ws (WebSocket) Ingress configuration + ws: + # paths configures ingress paths + paths: + - path: /ws + backendSuffix: gateway + # annotations defines k8s annotations to add to the Ingress + annotations: + nginx.ingress.kubernetes.io/proxy-send-timeout: "86400" + nginx.ingress.kubernetes.io/proxy-read-timeout: "86400" + nginx.ingress.kubernetes.io/proxy-next-upstream: "error timeout http_502" + nginx.ingress.kubernetes.io/proxy-next-upstream-tries: "4" + nginx.ingress.kubernetes.io/proxy-next-upstream-timeout: "50" + nginx.ingress.kubernetes.io/proxy-buffering: "off" + # the / Ingress configuration for serving the landing page and static resources + root: + # paths configures ingress paths + paths: + - path: / + pathType: Exact + backendSuffix: nginx + - path: /index.html + pathType: Exact + backendSuffix: nginx + - path: /ditto-up.svg + pathType: Exact + backendSuffix: nginx + - path: /ditto-down.svg + pathType: Exact + backendSuffix: nginx + # annotations defines k8s annotations to add to the Ingress + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: "10" + nginx.ingress.kubernetes.io/configuration-snippet: | + # security relevant headers: + add_header "Content-Security-Policy" "default-src 'self'; script-src-elem 'self' 'sha256-Kq9eqc/CtX2tgHPLJUEf8vDO9eNiGaRBrwAYYXTroVc=' https://cdnjs.cloudflare.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://cdnjs.cloudflare.com; worker-src 'self' blob:; object-src 'none';" always; + add_header "Strict-Transport-Security" "max-age=63072000; includeSubdomains;" always; + add_header "Cache-Control" "no-cache" always; + add_header "X-Content-Type-Options" "nosniff" always; + add_header "X-Frame-Options" "SAMEORIGIN" always; + add_header "X-XSS-Protection" "1; mode=block" always; + # ui the /ui and /apidoc Ingress configuration + ui: + # paths configures ingress paths + paths: + - path: / + pathType: Exact + backendSuffix: nginx + - path: /apidoc(/|$)(.*) + backendSuffix: swaggerui + - path: /ui(/|$)(.*) + backendSuffix: dittoui + # annotations defines k8s annotations to add to the Ingress + annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/proxy-connect-timeout: "10" + nginx.ingress.kubernetes.io/configuration-snippet: | + # security relevant headers: + add_header "Content-Security-Policy" "default-src 'self'; script-src-elem 'self' 'sha256-Ve/Ec/6YDEeTc+9y+QCJ+e9OhyGWAj3bYxCzNGfOn6U=' 'sha256-Kq9eqc/CtX2tgHPLJUEf8vDO9eNiGaRBrwAYYXTroVc=' https://cdnjs.cloudflare.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://cdnjs.cloudflare.com; img-src 'self' data: https://raw.githubusercontent.com; font-src 'self' https://cdnjs.cloudflare.com; worker-src 'self' blob:; connect-src 'self' localhost http://localhost:8080; object-src 'none';" always; + add_header "Strict-Transport-Security" "max-age=63072000; includeSubdomains;" always; + add_header "Cache-Control" "no-cache" always; + add_header "X-Content-Type-Options" "nosniff" always; + add_header "X-Frame-Options" "SAMEORIGIN" always; + add_header "X-XSS-Protection" "1; mode=block" always; + # tls configures the TLS for ingress + tls: [] + # - secretName: ditto-tls + # hosts: + # - localhost + + +## ---------------------------------------------------------------------------- +## openshift configures the OpenShift deployment +openshift: + # enabled whether to deploy to OpenShift + enabled: false + # routes the OpenShift Routes + routes: + # enabled whether OpenShift routes are enabled + enabled: false + # annotations define k8s annotations to apply for the routes + annotations: {} + # host: "" + # targetPort configures the target port + targetPort: http + # tlsTermination: "edge" + # tlsInsecurePolicy: "Redirect" + # securityContext the security context for OpenShift + securityContext: {} + +## ---------------------------------------------------------------------------- +## akka holds the Akka actor configuration +## ref: https://doc.akka.io/docs/akka/current/typed/index.html +akka: + # actorSystemName defines the actor/cluster name of the Ditto cluster + actorSystemName: ditto-cluster + # remoting holds configuration for the Akka cluster remoting + remoting: + # port defines the Port to use for remoting + port: 2551 + # mgmthttp holds configuration for the Akka cluster management + mgmthttp: + # port defines the Port to use for akka http management + port: 8558 + +# Set "dittoTag" in order to specify another Ditto version to use for all Ditto services: +# you may also use "1" (for latest Ditto 1.x.x) or "1.5" (for latest Ditto 1.5.x) +# dittoTag: 3.3.0 + + +## ---------------------------------------------------------------------------- +## policies configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-policies.html +policies: + # enabled controls whether policies related resources should be created + enabled: true + # replicaCount configuration for policies + replicaCount: 1 + # updateStrategy configuration for policies + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # minReadySeconds configures the minimum number of seconds for which a newly created Pod should be ready without any + # of its containers crashing, for it to be considered available + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds + minReadySeconds: 10 + # additionalLabels configuration for policies + additionalLabels: {} + # additionalAnnotations configuration for policies + additionalAnnotations: {} + image: + # repository for the policies docker image + repository: docker.io/eclipse/ditto-policies + # tag for the policies docker image - overwrite to specify something else than Chart.AppVersion + # tag: 3.3.0 + # pullPolicy for the policies docker image + pullPolicy: IfNotPresent + # additionalJvmOptions JVM options to put into JAVA_TOOL_OPTIONS + additionalJvmOptions: "" + # systemProps used to define arbitrary system properties for policies service + # ref: https://www.eclipse.dev/ditto/installation-operating.html#configuration + systemProps: + # extraEnv to add arbitrary environment variable to policies container + extraEnv: + # - name: LOG_LEVEL_APPLICATION + # value: "DEBUG" + # resources configures the resources available/to use for the policies service + resources: + # cpu defines the "required" CPU of a node so that the service is placed there + cpu: 0.5 + # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s + memoryMi: 1024 + # jvm contains JVM specific scaling/tuning configuration of e.g. processors and garbage collector settings + jvm: + # activeProcessorCount defines how many processors the JVM should be configured to use + # this is e.g. relevant for the GC which calculates the amount of asynchronous threads for GC based on the processor count + activeProcessorCount: 2 + # heapRamPercentage defines how much memory of the configured "resources.memoryMi" can be used by the JVM heap space + # be aware that the JVM also requires memory for "off heap" (and also stack) space + the container needs memory as well + heapRamPercentage: 60 + # maxGcPauseMillis configures the used G1 GC "target for the maximum GC pause time" + # default (by JVM if not set): 200 + maxGcPauseMillis: 150 + # readinessProbe configuration for policies + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + readinessProbe: + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + # livenessProbe configuration for policies + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + livenessProbe: + initialDelaySeconds: 160 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 4 + # podDisruptionBudget configuration for policies + # ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: + # enabled controls whether policies related PodDisruptionBudget should be created + enabled: true + # minAvailable number of replicas during voluntary disruptions + minAvailable: 1 + # nodeSelector configuration for policies + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + # tolerations configuration for policies + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + # affinity configuration for policies + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + # podMonitor configuration for policies + podMonitor: + # enabled configures whether Pod Monitor is enabled, then a resource to scrape policies metrics will be created + enabled: false + # interval: 30s + # scrapeTimeout: 15s + # config holds policies specific configuration + config: + # mongodb holds mongodb specific configuration of policies + mongodb: + # minPoolSize configures the minimum number of connections in the connection pool + minPoolSize: 10 + # maxPoolSize configures the minimum number of connections in the connection pool + maxPoolSize: 200 + # maxPoolIdleTime configures the maximum amount of time a pooled connection is allowed to idle before closing the connection + maxPoolIdleTime: 10m + # journalWriteConcern the MongoDB write concern to apply for writing operations on the event journal + # one of: Unacknowledged | Acknowledged | Journaled | ReplicaAcknowledged + journalWriteConcern: "Journaled" + # snapsWriteConcern the MongoDB write concern to apply for writing operations on the snapshots persistence + # one of: Unacknowledged | Acknowledged | Journaled | ReplicaAcknowledged + snapsWriteConcern: "Journaled" + # journalCircuitBreaker configures the circuit breaker for MongoDB operations on the event journal + journalCircuitBreaker: + # maxTries opens the circuit breaker if an exception during persisting an event occurs this often + # a successful write resets the counter + maxTries: 10 + # timeout configures the MongoDB write timeouts also causing the circuit breaker to open + timeout: 10s + # reset after this time in "Open" state, the circuit breaker is "Half-opened" again + reset: 5s + # snapsCircuitBreaker configures the circuit breaker for MongoDB operations on the snapshots persistence + snapsCircuitBreaker: + # maxTries opens the circuit breaker if an exception during persisting a snapshot occurs this often + # a successful write resets the counter + maxTries: 10 + # timeout configures the MongoDB write timeouts also causing the circuit breaker to open + timeout: 20s + # reset after this time in "Open" state, the circuit breaker is "Half-opened" again + reset: 8s + # cleanup contains the configuration for the background cleanup of stale snapshots and events + cleanup: + # enabled configures whether background cleanup is enabled or not + # if enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process: + enabled: false + # quietPeriod defines how long to stay in a state where the background cleanup is not yet started + quietPeriod: 5m + # history contains configuration regarding the event history + history: + # retentionDuration configures the duration of how long to "keep" events and snapshots before being allowed to remove them in scope of cleanup + retentionDuration: 30d + # metricsReporter config of MongoMetricsReporter which is used by policies in order to report current persistence + # roundtrip times in order to determine credits to cleanup stale data (journal entries, snapshots) + metricsReporter: + # resolution configures how far apart each measurement should be done + resolution: 1s + # history configures how many historical items to keep + history: 5 + # interval configures how often a "credit decision" is made + interval: 1s + # timerThreshold configures the maximum database latency to give out credit for cleanup actions + timerThreshold: 100ms + # creditsPerBatch configures how many "cleanup credits" should be generated per "interval" as long as the + creditsPerBatch: 5 + # persistence holds configuration regarding (akka) persistence of policies (event journal and snapshots) + persistence: + # activityCheckInterval configures to keep policies for that amount of time in memory when no other use did happen: + activityCheckInterval: 2d + # pingRate used to throttle pinging of PolicyPersistenceActors, so that not all PolicyPersistenceActors are recovered at the same time: + pingRate: + # frequency the frequency of sent "pings" to PolicyPersistenceActors + frequency: 1s + # entities the amount of entities to wake up per "frequency" interval + entities: 50 + # events contains event journal specific configuration + events: + # historicalHeadersToPersist define the DittoHeaders to persist when persisting events to the journal + # those can e.g. be retrieved as additional "audit log" information when accessing a historical Policy revision + historicalHeadersToPersist: + # - "ditto-originator" + # - "ditto-origin" + # - "correlation-id" + # snapshots contains snapshots persistence specific configuration + snapshots: + # interval configures the interval when to do snapshot for a Policy which had changes to it + interval: 15m + # threshold configures the threshold after how many changes to a Policy to do a snapshot + threshold: 5 + # entityCreation by default, Ditto allows anyone to create a new entity (policy in this case) in any namespace. + # However, this behavior can be customized, and the ability to create new entities can be restricted: + entityCreation: + # grants contains the list of creation config entries which would allow the creation of entities + # An empty list would *not* allow any entity to be created. + # You must have at least one entry, even if it is without restrictions. + grants: + - # namespaces holds the list of namespaces this entry applies to. An empty list would match any. + # Wildcards `*` (Matching any number of any character) and `?` (Matches any single character) are supported in entries of this list. + namespaces: [] + # authSubjects holds list of authentication subjects this entry applies to. An empty list would match any. + # Wildcards `*` (Matching any number of any character) and `?` (Matches any single character) are supported in entries of this list. + authSubjects: [] + # revokes contains the list of creation config entries which would reject the creation of entities + revokes: [] + # - namespaces: [] + # authSubjects: [] + +## ---------------------------------------------------------------------------- +## things configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-things.html +things: + # enabled controls whether things related resources should be created + enabled: true + # replicaCount configuration for things + replicaCount: 1 + # updateStrategy configuration for things + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # minReadySeconds configures the minimum number of seconds for which a newly created Pod should be ready without any + # of its containers crashing, for it to be considered available + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds + minReadySeconds: 10 + # additionalLabels configuration for things + additionalLabels: {} + # additionalAnnotations configuration for things + additionalAnnotations: {} + image: + # repository for the things docker image + repository: docker.io/eclipse/ditto-things + # tag for the things docker image - overwrite to specify something else than Chart.AppVersion + # tag: 3.3.0 + # pullPolicy for the things docker image + pullPolicy: IfNotPresent + # additionalJvmOptions JVM options to put into JAVA_TOOL_OPTIONS + additionalJvmOptions: "" + # systemProps used to define arbitrary system properties for things service + # ref: https://www.eclipse.dev/ditto/installation-operating.html#configuration + systemProps: + # extraEnv to add arbitrary environment variable to things container + extraEnv: + # - name: LOG_LEVEL_APPLICATION + # value: "DEBUG" + # resources configures the resources available/to use for the things service + resources: + # cpu defines the "required" CPU of a node so that the service is placed there + cpu: 0.5 + # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s + memoryMi: 1024 + # jvm contains JVM specific scaling/tuning configuration of e.g. processors and garbage collector settings + jvm: + # activeProcessorCount defines how many processors the JVM should be configured to use + # this is e.g. relevant for the GC which calculates the amount of asynchronous threads for GC based on the processor count + activeProcessorCount: 2 + # heapRamPercentage defines how much memory of the configured "resources.memoryMi" can be used by the JVM heap space + # be aware that the JVM also requires memory for "off heap" (and also stack) space + the container needs memory as well + heapRamPercentage: 60 + # maxGcPauseMillis configures the used G1 GC "target for the maximum GC pause time" + # default (by JVM if not set): 200 + maxGcPauseMillis: 150 + # readinessProbe configuration for things + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + readinessProbe: + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + # livenessProbe configuration for things + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + livenessProbe: + initialDelaySeconds: 160 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 4 + # podDisruptionBudget configuration for things + # ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: + # enabled controls whether things related PodDisruptionBudget should be created + enabled: true + # minAvailable number of replicas during voluntary disruptions + minAvailable: 1 + # nodeSelector configuration for things + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + # tolerations configuration for things + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + # affinity configuration for things + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + # podMonitor configuration for things + podMonitor: + # enabled configures whether Pod Monitor is enabled, then a resource to scrape things metrics will be created + enabled: false + # interval: 30s + # scrapeTimeout: 15s + # config holds things specific configuration + config: + # mongodb holds mongodb specific configuration of things + mongodb: + # minPoolSize configures the minimum number of connections in the connection pool + minPoolSize: 10 + # maxPoolSize configures the minimum number of connections in the connection pool + maxPoolSize: 200 + # maxPoolIdleTime configures the maximum amount of time a pooled connection is allowed to idle before closing the connection + maxPoolIdleTime: 10m + # journalWriteConcern the MongoDB write concern to apply for writing operations on the event journal + # one of: Unacknowledged | Acknowledged | Journaled | ReplicaAcknowledged + journalWriteConcern: "Acknowledged" + # snapsWriteConcern the MongoDB write concern to apply for writing operations on the snapshots persistence + # one of: Unacknowledged | Acknowledged | Journaled | ReplicaAcknowledged + snapsWriteConcern: "Acknowledged" + # journalCircuitBreaker configures the circuit breaker for MongoDB operations on the event journal + journalCircuitBreaker: + # maxTries opens the circuit breaker if an exception during persisting an event occurs this often + # a successful write resets the counter + maxTries: 10 + # timeout configures the MongoDB write timeouts also causing the circuit breaker to open + timeout: 10s + # reset after this time in "Open" state, the circuit breaker is "Half-opened" again + reset: 5s + # snapsCircuitBreaker configures the circuit breaker for MongoDB operations on the snapshots persistence + snapsCircuitBreaker: + # maxTries opens the circuit breaker if an exception during persisting a snapshot occurs this often + # a successful write resets the counter + maxTries: 10 + # timeout configures the MongoDB write timeouts also causing the circuit breaker to open + timeout: 20s + # reset after this time in "Open" state, the circuit breaker is "Half-opened" again + reset: 8s + # cleanup contains the configuration for the background cleanup of stale snapshots and events + cleanup: + # enabled configures whether background cleanup is enabled or not + # if enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process: + enabled: true + # quietPeriod defines how long to stay in a state where the background cleanup is not yet started + quietPeriod: 5m + # history contains configuration regarding the event history + history: + # retentionDuration configures the duration of how long to "keep" events and snapshots before being allowed to remove them in scope of cleanup + retentionDuration: 30d + # metricsReporter config of MongoMetricsReporter which is used by policies in order to report current persistence + # roundtrip times in order to determine credits to cleanup stale data (journal entries, snapshots) + metricsReporter: + # resolution configures how far apart each measurement should be done + resolution: 1s + # history configures how many historical items to keep + history: 5 + # interval configures how often a "credit decision" is made + interval: 1s + # timerThreshold configures the maximum database latency to give out credit for cleanup actions + timerThreshold: 100ms + # creditsPerBatch configures how many "cleanup credits" should be generated per "interval" as long as the + creditsPerBatch: 5 + # persistence holds configuration regarding (akka) persistence of things (event journal and snapshots) + persistence: + # activityCheckInterval configures to keep things for that amount of time in memory when no other use did happen + activityCheckInterval: 2d + # events contains event journal specific configuration + events: + # historicalHeadersToPersist define the DittoHeaders to persist when persisting events to the journal + # those can e.g. be retrieved as additional "audit log" information when accessing a historical Thing revision + historicalHeadersToPersist: + # - "ditto-originator" + # - "ditto-origin" + # - "correlation-id" + # snapshots contains snapshots persistence specific configuration + snapshots: + # the interval when to do snapshot for a Thing which had changes to it + interval: 15m + # the threshold after how many changes to a Thing to do a snapshot + threshold: 50 + # entityCreation by default, Ditto allows anyone to create a new entity (thing in this case) in any namespace. + # However, this behavior can be customized, and the ability to create new entities can be restricted: + entityCreation: + # grants contains the list of creation config entries which would allow the creation of entities + # An empty list would *not* allow any entity to be created. + # You must have at least one entry, even if it is without restrictions. + grants: + - # namespaces holds the list of namespaces this entry applies to. An empty list would match any. + # Wildcards `*` (Matching any number of any character) and `?` (Matches any single character) are supported in entries of this list. + namespaces: [] + # authSubjects holds list of authentication subjects this entry applies to. An empty list would match any. + # Wildcards `*` (Matching any number of any character) and `?` (Matches any single character) are supported in entries of this list. + authSubjects: [] + # revokes contains the list of creation config entries which would reject the creation of entities + revokes: [] + # - namespaces: [] + # authSubjects: [] + # policiesEnforcer contains configuration for Ditto "Policy Enforcers", e.g. regarding caching + policiesEnforcer: + # cache holds the configuration of policy enforcer caching + cache: + # enabled whether caching of policy enforcers should be enabled + enabled: true + # maxSize the maximum size of policy enforcers to keep in the cache + maxSize: 50000 + # expireAfterWrite the maximum duration of inconsistency after losing a cache invalidation + expireAfterWrite: 8h + # expireAfterAccess prolonged on each cache access by that duration + expireAfterAccess: 4h + # wot contains Web of Things (WoT) specific configuration + wot: + # tdBasePrefix is the base to use where the Ditto endpoint is located in order to be injected into TDs: + tdBasePrefix: "http://localhost:8080" + # tdJsonTemplate contains a json template added to generated TDs, e.g. containing security information: + tdJsonTemplate: >- + { + "securityDefinitions": { + "basic_sc": { + "scheme": "basic", + "in": "header" + } + }, + "security": "basic_sc", + "support": "https://www.eclipse.dev/ditto/" + } + +## ---------------------------------------------------------------------------- +## things-search configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-things-search.html +thingsSearch: + # enabled controls whether things-search related resources should be created + enabled: true + # replicaCount configuration for things-search + replicaCount: 1 + # updateStrategy configuration for things-search + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # minReadySeconds configures the minimum number of seconds for which a newly created Pod should be ready without any + # of its containers crashing, for it to be considered available + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds + minReadySeconds: 10 + # additionalLabels configuration for things-search + additionalLabels: {} + # additionalAnnotations configuration for things-search + additionalAnnotations: {} + image: + # repository for the things-search docker image + repository: docker.io/eclipse/ditto-things-search + # tag for the things-search docker image - overwrite to specify something else than Chart.AppVersion + # tag: 3.3.0 + # pullPolicy for the things-search docker image + pullPolicy: IfNotPresent + # additional JVM options to put into JAVA_TOOL_OPTIONS + additionalJvmOptions: "" + # systemProps used to define arbitrary system properties for things-search service + # ref: https://www.eclipse.dev/ditto/installation-operating.html#configuration + systemProps: + # extraEnv to add arbitrary environment variable to things-search container + extraEnv: + # - name: LOG_LEVEL_APPLICATION + # value: "DEBUG" + # resources configures the resources available/to use for the things search service + resources: + # cpu defines the "required" CPU of a node so that the service is placed there + cpu: 0.5 + # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s + memoryMi: 1024 + # jvm contains JVM specific scaling/tuning configuration of e.g. processors and garbage collector settings + jvm: + # activeProcessorCount defines how many processors the JVM should be configured to use + # this is e.g. relevant for the GC which calculates the amount of asynchronous threads for GC based on the processor count + activeProcessorCount: 2 + # heapRamPercentage defines how much memory of the configured "resources.memoryMi" can be used by the JVM heap space + # be aware that the JVM also requires memory for "off heap" (and also stack) space + the container needs memory as well + heapRamPercentage: 60 + # maxGcPauseMillis configures the used G1 GC "target for the maximum GC pause time" + # default (by JVM if not set): 200 + maxGcPauseMillis: 150 + # readinessProbe configuration for things-search + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + readinessProbe: + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + # livenessProbe configuration for things-search + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + livenessProbe: + initialDelaySeconds: 160 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 4 + # podDisruptionBudget configuration for things-search + # ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: + # enabled controls whether things-search related PodDisruptionBudget should be created + enabled: true + # minAvailable number of replicas during voluntary disruptions + minAvailable: 1 + # nodeSelector configuration for things-search + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + # tolerations configuration for things-search + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + # affinity configuration for things-search + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + # podMonitor configuration for things-search + podMonitor: + # enabled configures whether Pod Monitor is enabled, then a resource to scrape things search metrics will be created + enabled: false + # interval: 30s + # scrapeTimeout: 15s + # config holds things-search specific configuration + config: + # mongodb holds mongodb specific configuration of things-search + mongodb: + # minPoolSize configures the minimum number of connections in the connection pool + minPoolSize: 10 + # maxPoolSize configures the minimum number of connections in the connection pool + maxPoolSize: 100 + # maxPoolIdleTime configures the maximum amount of time a pooled connection is allowed to idle before closing the connection + maxPoolIdleTime: 10m + # searchReadPreference configures the overall MongoDB read preference + # one of: primary | primaryPreferred | secondary | secondaryPreferred | nearest + searchReadPreference: "primary" + # searchWriteConcern configures the overall MongoDB write concern + # one of: unacknowledged | acknowledged | majority | journaled | w1 | w2 | w3 + searchWriteConcern: "acknowledged" + # searchWithAcksWriteConcern configures the MongoDB write concern for commands sent with "search-persisted" ACK + # ref: https://www.eclipse.dev/ditto/basic-acknowledgements.html#built-in-acknowledgement-labels + # one of: unacknowledged | acknowledged | majority | journaled | w1 | w2 | w3 + searchWithAcksWriteConcern: "majority" + # queryReadConcern configures the MongoDB read concern for doing queries / performing searches + # only if this is "linearizable" in combination with the searchWithAcksWriteConcern: "majority" a strong consistency + # if used in a replicated MongoDB setup, this should be changed to `queryReadConcern: "linearizable"` + # for commands using the "search-persisted" requested ACK is guaranteed + # one of: default | local | majority | linearizable | snapshot | available + queryReadConcern: "local" + # updaterPersistenceReadConcern configures the MongoDB read concern for the "ThingUpdater" + # one of: default | local | majority | linearizable | snapshot | available + updaterPersistenceReadConcern: "local" + # updaterPersistenceReadPreference configures the MongoDB read preference for the "ThingUpdater" + updaterPersistenceReadPreference: "primaryPreferred" + # updater contains configuration for the "Things Updater" of things-search service + updater: + # activityCheckInterval configures to keep thing updaters for that amount of time in memory when no update did happen: + activityCheckInterval: 2h + # stream contains streaming configuration settings of the things-search service + stream: + # retrievalParallelism configures the upper bound of parallel SudoRetrieveThing commands + # (by extension, parallel loads of policy enforcer cache) + retrievalParallelism: 64 + persistence: + # parallelism configures how much bulk writes to request in parallel - must be a power of 2 + parallelism: 16 + # policiesEnforcer contains configuration for Ditto "Policy Enforcers", e.g. regarding caching + policiesEnforcer: + # cache holds the configuration of policy enforcer caching + cache: + # maxSize the maximum size of policy enforcers to keep in the cache + maxSize: 30000 + # expireAfterWrite the maximum duration of inconsistency after losing a cache invalidation + expireAfterWrite: 12h + # expireAfterAccess prolonged on each cache access by that duration + expireAfterAccess: 6h + # thingCache configures the cache configuration for caching of things in things-search + thingCache: + # maxSize defines how many things to cache + maxSize: 30000 + # expireAfterWrite defines how long at most to keep things in the cache after loading them into the cache + expireAfterWrite: 12h + # expireAfterWrite defines how long at most to keep things in the cache after last accessing them from the cache + expireAfterAccess: 6h + # backgroundSync contains the configuration for the "background sync" responsible for continuously streaming + # over snapshot entries of things to ensure the eventual consistency of the search index + backgroundSync: + # enabled whether background sync is turned on + enabled: true + # quietPeriod the duration between service start-up and the beginning of background sync + quietPeriod: 5m + # idleTimeout how soon to close the remote stream if no element passed through it + idleTimeout: 5m + # toleranceWindow how long to wait before reacting to out-of-date search index entries + toleranceWindow: 20m + # keepEvents how many events to keep in the actor state + keepEvents: 2 + # throttle contains the background sync throttling configuration + throttle: + # throughput how many things to update per throttle period + throughput: 100 + # period the throttle period + period: 30s + + +## ---------------------------------------------------------------------------- +## connectivity configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-connectivity.html +connectivity: + # enabled controls whether connectivity related resources should be created + enabled: true + # replicaCount configuration for connectivity + replicaCount: 1 + # updateStrategy configuration for connectivity + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # minReadySeconds configures the minimum number of seconds for which a newly created Pod should be ready without any + # of its containers crashing, for it to be considered available + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds + minReadySeconds: 10 + # additionalLabels configuration for connectivity + additionalLabels: {} + # additionalAnnotations configuration for connectivity + additionalAnnotations: {} + image: + # repository for the connectivity docker image + repository: docker.io/eclipse/ditto-connectivity + # tag for the connectivity docker image - overwrite to specify something else than Chart.AppVersion + # tag: 3.3.0 + # pullPolicy for the connectivity docker image + pullPolicy: IfNotPresent + # additional JVM options to put into JAVA_TOOL_OPTIONS + additionalJvmOptions: "" + # systemProps used to define arbitrary system properties for connectivity service + # ref: https://www.eclipse.dev/ditto/installation-operating.html#configuration + systemProps: + # extraEnv to add arbitrary environment variable to connectivity container + extraEnv: + # - name: LOG_LEVEL_APPLICATION + # value: "DEBUG" + # resources configures the resources available/to use for the connectivity service + resources: + # cpu defines the "required" CPU of a node so that the service is placed there + cpu: 0.5 + # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s + memoryMi: 1024 + # jvm contains JVM specific scaling/tuning configuration of e.g. processors and garbage collector settings + jvm: + # activeProcessorCount defines how many processors the JVM should be configured to use + # this is e.g. relevant for the GC which calculates the amount of asynchronous threads for GC based on the processor count + activeProcessorCount: 2 + # heapRamPercentage defines how much memory of the configured "resources.memoryMi" can be used by the JVM heap space + # be aware that the JVM also requires memory for "off heap" (and also stack) space + the container needs memory as well + heapRamPercentage: 60 + # maxGcPauseMillis configures the used G1 GC "target for the maximum GC pause time" + # default (by JVM if not set): 200 + maxGcPauseMillis: 150 + # readinessProbe configuration for connectivity + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + readinessProbe: + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + # livenessProbe configuration for connectivity + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + livenessProbe: + initialDelaySeconds: 160 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 4 + # podDisruptionBudget configuration for connectivity + # ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: + # enabled controls whether connectivity related PodDisruptionBudget should be created + enabled: true + # minAvailable number of replicas during voluntary disruptions + minAvailable: 1 + # nodeSelector configuration for connectivity + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + # tolerations configuration for connectivity + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + # affinity configuration for connectivity + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + # podMonitor configuration for connectivity + podMonitor: + # enabled configures whether Pod Monitor is enabled, then a resource to scrape connectivity metrics will be created + enabled: false + # interval: 30s + # scrapeTimeout: 15s + # config holds connectivity specific configuration + config: + # mongodb holds mongodb specific configuration of connectivity + mongodb: + # minPoolSize configures the minimum number of connections in the connection pool + minPoolSize: 10 + # maxPoolSize configures the minimum number of connections in the connection pool + maxPoolSize: 50 + # maxPoolIdleTime configures the maximum amount of time a pooled connection is allowed to idle before closing the connection + maxPoolIdleTime: 10m + # journalWriteConcern the MongoDB write concern to apply for writing operations on the event journal + # one of: Unacknowledged | Acknowledged | Journaled | ReplicaAcknowledged + journalWriteConcern: "Journaled" + # snapsWriteConcern the MongoDB write concern to apply for writing operations on the snapshots persistence + # one of: Unacknowledged | Acknowledged | Journaled | ReplicaAcknowledged + snapsWriteConcern: "Journaled" + # journalCircuitBreaker configures the circuit breaker for MongoDB operations on the event journal + journalCircuitBreaker: + # maxTries opens the circuit breaker if an exception during persisting an event occurs this often + # a successful write resets the counter + maxTries: 10 + # timeout configures the MongoDB write timeouts also causing the circuit breaker to open + timeout: 10s + # reset after this time in "Open" state, the circuit breaker is "Half-opened" again + reset: 5s + # snapsCircuitBreaker configures the circuit breaker for MongoDB operations on the snapshots persistence + snapsCircuitBreaker: + # maxTries opens the circuit breaker if an exception during persisting a snapshot occurs this often + # a successful write resets the counter + maxTries: 10 + # timeout configures the MongoDB write timeouts also causing the circuit breaker to open + timeout: 20s + # reset after this time in "Open" state, the circuit breaker is "Half-opened" again + reset: 8s + # policiesEnforcer contains configuration for Ditto "Policy Enforcers", e.g. regarding caching + policiesEnforcer: + # cache holds the configuration of policy enforcer caching + cache: + # enabled whether caching of policy enforcers should be enabled + enabled: true + # maxSize the maximum size of policy enforcers to keep in the cache + maxSize: 1000 + # expireAfterWrite the maximum duration of inconsistency after losing a cache invalidation + expireAfterWrite: 8h + # expireAfterAccess prolonged on each cache access by that duration + expireAfterAccess: 4h + # cleanup contains the configuration for the background cleanup of stale snapshots and events + cleanup: + # enabled configures whether background cleanup is enabled or not + # if enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process: + enabled: false + # quietPeriod defines how long to stay in a state where the background cleanup is not yet started + quietPeriod: 5m + # history contains configuration regarding the event history + history: + # retentionDuration configures the duration of how long to "keep" events and snapshots before being allowed to remove them in scope of cleanup + retentionDuration: 30d + # metricsReporter config of MongoMetricsReporter which is used by policies in order to report current persistence + # roundtrip times in order to determine credits to cleanup stale data (journal entries, snapshots) + metricsReporter: + # resolution configures how far apart each measurement should be done + resolution: 1s + # history configures how many historical items to keep + history: 5 + # interval configures how often a "credit decision" is made + interval: 10s + # timerThreshold configures the maximum database latency to give out credit for cleanup actions + timerThreshold: 100ms + # creditsPerBatch configures how many "cleanup credits" should be generated per "interval" as long as the + creditsPerBatch: 5 + # persistence holds configuration regarding (akka) persistence of connections (event journal and snapshots) + persistence: + # keep closed, inactive connections for that amount of time in memory when no other use did happen: + activityCheckInterval: 45m + # events contains event journal specific configuration + events: + # historicalHeadersToPersist define the DittoHeaders to persist when persisting events to the journal + # those can e.g. be retrieved as additional "audit log" information when accessing a historical Connection revision + historicalHeadersToPersist: + # - "ditto-originator" + # - "ditto-origin" + # - "correlation-id" + # snapshots contains snapshots persistence specific configuration + snapshots: + # interval the interval when to do snapshot for a Connection which had changes to it + interval: 15m + # threshold the threshold after how many changes to a Connection to do a snapshot + threshold: 5 + # connections holds configuration regarding connections + connections: + # reconnect configures pinging of connections, so that not all connections are recovered at the same time + reconnect: + # rate configures the rate in which frequency to ping/wake up how many entities (connections) + rate: + # frequency the frequency of how often to wake up connections after restart + frequency: 1s + # entities the amount of entities to wake up per "frequency" interval + entities: 10 + # allowedHostnames contains a comma separated list of explicitly allowed hostnames + allowedHostnames: "" + # blockedHostnames contains a comma separated list of blocked hostnames + blockedHostnames: "" + # blockedSubnets holds a comma separated string of blocked subnets + # specify subnets to block in CIDR format e.g. "11.1.0.0/16" + blockedSubnets: "" + # blockedHostRegex contains the regex for blocked hostnames + blockedHostRegex: "" + limits: + # maxSources contains the max number of sources per connection + maxSources: 5 + # maxTargets contains the max number of targets per connection + maxTargets: 5 + enrichment: + # the buffer size used for the queue in the message mapping processor actor + bufferSize: 200 + # kafka contains the configuration specific to Ditto connections to Apache Kafka + kafka: + # consumer contains configuration for consuming messages from Kafka + consumer: + # throttling contains configuration for applying throttling when consuming from a single Kafka connection + throttling: + # enabled defines whether throttling should be applied when consuming messages from a Kafka source + enabled: true + # interval the interval at which the consumer is throttled - must be > 0s + interval: 1s + # limit defines the maximum number of messages the consumer is allowed to receive within the configured + # throttling "interval" e.g. 100 msgs/s - must be > 0 + limit: 500 + # maxInflightFactor configures how many unacknowledged messages are allowed at any time as factor of + # ${limit} - must be >= 1.0 + # This limit couples latency with throughput (long latency before ack -> lower throughput) + maxInflightFactor: 2.0 + # producer contains configuration for publishing messages to Kafka + producer: + # If a message can't be published it is put in a queue. Further messages are dropped when the queue is full. + queueSize: 1000 + # Messages to publish in parallel per Kafka-Publisher (one per connectivity client) + parallelism: 10 + +## ---------------------------------------------------------------------------- +## gateway configuration +## ref: https://www.eclipse.dev/ditto/architecture-services-gateway.html +gateway: + # enabled controls whether gateway related resources should be created + enabled: true + # replicaCount configuration for gateway + replicaCount: 1 + # updateStrategy configuration for gateway + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # minReadySeconds configures the minimum number of seconds for which a newly created Pod should be ready without any + # of its containers crashing, for it to be considered available + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds + minReadySeconds: 10 + # additionalLabels configuration for gateway + additionalLabels: {} + # additionalAnnotations configuration for gateway + additionalAnnotations: {} + # additional JVM options to put into JAVA_TOOL_OPTIONS + additionalJvmOptions: "" + image: + # repository for the gateway docker image + repository: docker.io/eclipse/ditto-gateway + # tag for the gateway docker image - overwrite to specify something else than Chart.AppVersion + # tag: 3.3.0 + # pullPolicy for the gateway docker image + pullPolicy: IfNotPresent + # systemProps used to define arbitrary system properties configuration for gateway + # ref: https://www.eclipse.dev/ditto/installation-operating.html#configuration + systemProps: + - "-Dditto.protocol.blocklist.0=raw-request-uri" + - "-Dditto.protocol.blocklist.1=cache-control" + - "-Dditto.protocol.blocklist.2=connection" + - "-Dditto.protocol.blocklist.3=timeout-access" + - "-Dditto.protocol.blocklist.4=accept-encoding" + - "-Dditto.protocol.blocklist.5=x-forwarded-scheme" + - "-Dditto.protocol.blocklist.6=x-forwarded-port" + - "-Dditto.protocol.blocklist.7=x-forwarded-for" + - "-Dditto.protocol.blocklist.8=forwarded=for" + - "-Dditto.protocol.blocklist.9=sec-fetch-mode" + - "-Dditto.protocol.blocklist.10=sec-fetch-site" + - "-Dditto.protocol.blocklist.11=authorization" + - "-Dditto.protocol.blocklist.12=accept-language" + - "-Dditto.protocol.blocklist.13=host" + - "-Dditto.protocol.blocklist.14=via" + - "-Dditto.protocol.blocklist.15=sec-ch-ua" + - "-Dditto.protocol.blocklist.16=sec-ch-ua-mobile" + - "-Dditto.protocol.blocklist.17=sec-ch-ua-platform" + - "-Dditto.protocol.blocklist.18=sec-fetch-dest" + - "-Dditto.protocol.blocklist.19=user-agent" + # extraEnv to add arbitrary environment variables to gateway container + extraEnv: + # - name: LOG_LEVEL_APPLICATION + # value: "DEBUG" + # resources configures the resources available/to use for the gateway service + resources: + # cpu defines the "required" CPU of a node so that the service is placed there + cpu: 0.5 + # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s + memoryMi: 1024 + # jvm contains JVM specific scaling/tuning configuration of e.g. processors and garbage collector settings + jvm: + # activeProcessorCount defines how many processors the JVM should be configured to use + # this is e.g. relevant for the GC which calculates the amount of asynchronous threads for GC based on the processor count + activeProcessorCount: 2 + # heapRamPercentage defines how much memory of the configured "resources.memoryMi" can be used by the JVM heap space + # be aware that the JVM also requires memory for "off heap" (and also stack) space + the container needs memory as well + heapRamPercentage: 60 + # maxGcPauseMillis configures the used G1 GC "target for the maximum GC pause time" + # default (by JVM if not set): 200 + maxGcPauseMillis: 150 + # readinessProbe configuration for gateway + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + readinessProbe: + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + # livenessProbe configuration for gateway + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + livenessProbe: + initialDelaySeconds: 160 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 4 + # service configuration of the k8s service of the gateway + service: + # port number configuration for gateway + port: 8080 + # annotations to add arbitrary annotations to nginx service + annotations: {} + # podDisruptionBudget configuration for gateway + # ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: + # enabled controls whether gateway related PodDisruptionBudget should be created + enabled: true + # minAvailable number of replicas during voluntary disruptions + minAvailable: 1 + # nodeSelector configuration for gateway + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + # tolerations configuration for gateway + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + # affinity configuration for gateway + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + # podMonitor configuration for gateway + podMonitor: + # enabled configures whether Pod Monitor is enabled, then a resource to scrape gateway metrics will be created + enabled: false + # interval: 30s + # scrapeTimeout: 15s + # config holds gateway specific configuration + config: + # authentication contains the settings regarding authentication against the gateway + authentication: + # enablePreAuthentication whether Ditto "pre-authentication" should be enabled + # ref: https://www.eclipse.dev/ditto/installation-operating.html#pre-authentication + enablePreAuthentication: false + # oauth contains the OAuth2.0 / OpenID Connect related configuration + oauth: + # allowedClockSkew configures the amount of clock skew in seconds to tolerate when verifying the local time against the exp and nbf claims + allowedClockSkew: 20s + # openidConnectIssuers holds a map of issuer-prefixes as key (e.g. "example") + # and OAuth "issuer" and "authSubjects" list containing which claims to extract from a JWT issued by the issuer + openidConnectIssuers: + # example: + # issuer: "example.com" + # authSubjects: + # - "{{ jwt:sub }}" + # - "{{ jwt:groups }}" + # devops contains the configuration of the gateway's "/devops" API, e.g. access to it + devops: + # secured this controls whether "/devops" and "/api/2/connections" resources are secured or not + secured: true + # authMethod declares the authentication method to apply for authenticating the "/devops" and "/api/2/connections" resources + # one of: "basic" | "oauth2" + authMethod: "basic" + # oauth contains the OAuth2.0 / OpenID Connect related configuration applied when "authMethod" above is "oauth2" + oauth: + # allowedClockSkew configures the amount of clock skew in seconds to tolerate when verifying the local time against the exp and nbf claims + allowedClockSkew: 20s + # openidConnectIssuers holds a map of issuer-prefixes as key (e.g. "example") + # and OAuth "issuer" and "authSubjects" list containing which claims to extract from a JWT issued by the issuer + openidConnectIssuers: + # example-ops: + # issuer: "example.com" + # authSubjects: + # - "{{ jwt:sub }}" + # - "{{ jwt:groups }}" + # oauthSubjects contains list of subjects authorized to use "/devops" and "/api/2/connections" resources + oauthSubjects: + # - "example-ops:devops-admin" + # statusSecured controls whether the "/status" and "/status/health" resources are secured or not + statusSecured: true + # statusAuthMethod declares the authentication method to apply for authenticating the "/status" and "/status/health" resources + # one of: "basic" | "oauth2" + statusAuthMethod: "basic" + # statusOauthSubjects contains list of subjects authorized to use "/status" API + statusOauthSubjects: + # - "example-ops:devops-admin" + # existingSecret contains the name of existing secret containing status and devops passwords + # if not set then default secret is created - useful for managing secrets with external secrets manager + existingSecret: + # devopsPassword the password to use for accessing "/devops" and "/api/2/connections" resources + # when "authMethod": "basic" (with username: devops) + # if not set a random password will be generated and used + devopsPassword: + # statusPassword will be used for accessing "/status" and "/status/health" resources + # when "statusAuthMethod": "basic" (with username: devops) + # if not set a random password will be set + statusPassword: + # websocket contains the gateway websocket configuration + websocket: + # subscriber contains the configuration for receiving data via the websocket + subscriber: + # backpressureQueueSize is the max queue size of how many inflight commands a single websocket client can have + backpressureQueueSize: 100 + # publisher contains the configuration for sending/publishing data via the websocket + publisher: + # backpressureBufferSize is the max buffer size of how many outstanding CommandResponses and Events a single + # websocket client can have - additional CommandResponses and Events are dropped if this size is reached + backpressureBufferSize: 200 + # throttling contains the throttling configuration of a single websocket session + throttling: + # enabled whether throttling message consumption via a single websocket session is enabled + enabled: true + # interval is the interval at which a single websocket session is rate-limited - must be > 0s + interval: 1s + # limit is the maximum number of messages the websocket session is allowed to receive within the configured + # throttling interval e.g. 100 msgs/s + limit: 100 + # websocket contains the gateway SSE (server sent events) configuration + sse: + # throttling contains the throttling configuration of a single SSE session (only applies for search via SSE) + throttling: + # enabled whether throttling message publishing via a single websocket session is enabled + enabled: true + # interval is the interval at which a single SSE session is rate-limited - must be > 0s + interval: 1s + # limit is the maximum number of messages the SSE session is allowed to receive within the configured + # throttling interval e.g. 100 msgs/s + limit: 100 + +## ---------------------------------------------------------------------------- +## nginx configuration +nginx: + # enabled controls whether nginx related resources should be created + enabled: true + # replicaCount for nginx + replicaCount: 1 + # updateStrategy for nginx + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # additionalLabels on nginx pods + additionalLabels: {} + # additionalAnnotations on nginx pods + additionalAnnotations: {} + image: + # repository for the nginx docker image + repository: docker.io/nginx + # tag for the nginx docker image + tag: 1.25 + # pullPolicy for the nginx docker image + pullPolicy: IfNotPresent + # extraEnv to add arbitrary environment variables to nginx container + extraEnv: [] + # resources configures the resources available/to use for nginx + resources: + # cpu defines the "required" CPU of a node so that the service is placed there + cpu: 0.2 + # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s + memoryMi: 64 + # readinessProbe configuration for nginx + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + readinessProbe: {} + # livenessProbe configuration for nginx + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes + livenessProbe: {} + # service configuration of the k8s service of the nginx + service: + # type of the nginx service + type: ClusterIP + # port of the nginx service + port: 8080 + # in case of NodePort the may additionally be set + # type: NodePort + # nodePort: 30080 + # annotations to add arbitrary annotations to nginx service + annotations: {} + # nodeSelector configuration for nginx + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + # tolerations configuration for nginx + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + # affinity configuration for nginx + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + # init containers for nginx + initContainers: + waitForGateway: + enabled: true + name: wait-for-gateway + image: rancher/curlimages-curl:7.73.0 + # config holds nginx specific configuration + config: + # workerProcesses the 'worker_processes' option for nginx to use - can also be set to 'auto' in order to let nginx + # determine the worker processes based on the CPU count + workerProcesses: 4 + # workerProcesses the 'events' 'worker_connections' option for nginx to use + workerConnections: 1024 + +## ---------------------------------------------------------------------------- +## Ditto UI configuration +dittoui: + enabled: true + # replicaCount for Ditto UI service + replicaCount: 1 + # updateStrategy for Ditto UI service + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # additionalLabels on Ditto UI pods + additionalLabels: {} + # additionalAnnotations on Ditto UI pods + additionalAnnotations: {} + image: + # repository for the Ditto UI docker image + repository: docker.io/eclipse/ditto-ui + # tag for the Ditto UI image - overwrite to specify something else than Chart.AppVersion + # tag: 3.3.0 + # pullPolicy for the Ditto UI docker image + pullPolicy: IfNotPresent + # extraEnv to add arbitrary environment variable to Ditto UI container + extraEnv: [] + # resources configures the resources available/to use for the Ditto UI container + resources: + # cpu defines the "required" CPU of a node so that the service is placed there + cpu: 0.1 + # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s + memoryMi: 64 + # podDisruptionBudget ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: + # enabled controls whether Ditto UI related PodDisruptionBudget should be created + enabled: true + # minAvailable number of replicas during voluntary disruptions + minAvailable: 1 + # service configuration of the k8s service of the Ditto UI + service: + # port of the Ditto UI service + port: 8080 + # annotations to add arbitrary annotations to Ditto UI service + annotations: {} + +## ---------------------------------------------------------------------------- +## swaggerui configuration +swaggerui: + # enabled controls whether swagger ui related resources should be created + enabled: true + # replicaCount for swagger ui service + replicaCount: 1 + # updateStrategy for swagger ui service + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # additionalLabels on swagger ui pods + additionalLabels: {} + # additionalAnnotations on swagger ui pods + additionalAnnotations: {} + image: + # repository for the swagger ui docker image + repository: docker.io/swaggerapi/swagger-ui + # tag for the swagger ui docker image + tag: v4.19.1 + # pullPolicy for the swagger ui docker image + pullPolicy: IfNotPresent + # extraEnv to add arbitrary environment variable to swagger ui container + extraEnv: [] + # resources configures the resources available/to use for the swagger ui container + resources: + # cpu defines the "required" CPU of a node so that the service is placed there + cpu: 0.1 + # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s + memoryMi: 64 + # podDisruptionBudget ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: + # enabled controls whether swagger ui related PodDisruptionBudget should be created + enabled: true + # minAvailable number of replicas during voluntary disruptions + minAvailable: 1 + # service configuration of the k8s service of the swagger ui + service: + # port of the swagger ui service + port: 8080 + # annotations to add arbitrary annotations to swagger ui service + annotations: {} + +## ---------------------------------------------------------------------------- +## mongodb dependency chart configuration +mongodb: + # enabled controls whether mongodb should be started as part of the Helm chart or not + enabled: true + # fullnameOverride: ditto-mongodb + auth: + enabled: false + securityContext: + enabled: false + persistence: + enabled: false diff --git a/deployment/kubernetes/README.md b/deployment/kubernetes/README.md index 4f34d10311f..351e404874f 100644 --- a/deployment/kubernetes/README.md +++ b/deployment/kubernetes/README.md @@ -53,7 +53,7 @@ In case you already have a MongoDB in the cloud or elsewhere it is possible to c This can be done by setting the MongoDB URI via env variable "MONGO_DB_URI" in the `deployment/kubernetes/deploymentFiles/ditto/ditto-cluster.yml` for all services except the `gateway`. Other MongoDB settings can be set via env variables and are documented in -[Operating Ditto](https://www.eclipse.org/ditto/installation-operating.html) section. +[Operating Ditto](https://www.eclipse.dev/ditto/installation-operating.html) section. In case your "MONGO_DB_URI" contains sensitive information like username and password it is recommended to use a kubernetes secret. diff --git a/deployment/openshift/nginx/index.html b/deployment/openshift/nginx/index.html index d836b023218..d2c8839941d 100644 --- a/deployment/openshift/nginx/index.html +++ b/deployment/openshift/nginx/index.html @@ -41,7 +41,7 @@

You have started Eclipse Ditto

Thank you for trying out Eclipse Ditto!

-

In order to get started quickly, you can now have a look at the documentation +

In order to get started quickly, you can now have a look at the documentation

To authenticate at the HTTP APIs use the following username "ditto" and password "ditto" when asked for by your browser.

diff --git a/deployment/openshift/nginx/nginx.conf b/deployment/openshift/nginx/nginx.conf index c33bc6b3a8b..8bd6cc45b01 100644 --- a/deployment/openshift/nginx/nginx.conf +++ b/deployment/openshift/nginx/nginx.conf @@ -1,6 +1,8 @@ -worker_processes 1; +worker_processes auto; -events {worker_connections 1024;} +events { + worker_connections 1024; +} http { charset utf-8; diff --git a/deployment/operations/grafana-dashboards/Akka.json b/deployment/operations/grafana-dashboards/Akka.json index 75d5b19244b..32d6e089eda 100644 --- a/deployment/operations/grafana-dashboards/Akka.json +++ b/deployment/operations/grafana-dashboards/Akka.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -29,10 +26,7 @@ "liveNow": false, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -130,10 +124,7 @@ "text": "None", "value": "" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(actor_mailbox_size, service)", "hide": 0, "includeAll": false, @@ -159,10 +150,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(actor_mailbox_size{service=~\"$service\"}, instance)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Akka_Dispatcher_Metrics.json b/deployment/operations/grafana-dashboards/Akka_Dispatcher_Metrics.json index f2dff2f4b2e..9b36113fb79 100644 --- a/deployment/operations/grafana-dashboards/Akka_Dispatcher_Metrics.json +++ b/deployment/operations/grafana-dashboards/Akka_Dispatcher_Metrics.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -34,10 +31,7 @@ { "collapse": false, "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -52,10 +46,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Tracks the number of threads in use.", "fieldConfig": { "defaults": { @@ -131,10 +122,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_threads_total_sum{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])\n/\nrate(executor_threads_total_count{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -144,10 +132,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_threads_active_sum{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])\n/\nrate(executor_threads_active_count{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -166,10 +151,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -211,10 +193,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_tasks_submitted_total{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -225,10 +204,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_tasks_completed_total{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -239,10 +215,7 @@ "refId": "B" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "max_over_time( ( rate( executor_tasks_completed_total{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval] ) )[$__interval:30s] )", "format": "time_series", @@ -284,10 +257,7 @@ } }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -366,10 +336,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "rate(executor_queue_size_sum{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])\n/\nrate(executor_queue_size_count{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", "hide": false, @@ -378,10 +345,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(executor_queue_size_bucket{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\",le=\"+Inf\"}[$Interval])) by (job,instance,name,le))", "format": "time_series", @@ -396,10 +360,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Tracks minimum/maximum number of Threads of the executors.", "fieldConfig": { "defaults": { @@ -479,10 +440,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "executor_threads_min{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}", "format": "time_series", @@ -492,10 +450,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "executor_threads_max{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}", "format": "time_series", @@ -514,10 +469,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Tracks executor parallelism.", "fill": 1, "fillGradient": 0, @@ -560,10 +512,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "executor_parallelism{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}", "format": "time_series", @@ -604,10 +553,7 @@ } }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Tracks the time that tasks spend on the executor service's queue", "fieldConfig": { "defaults": { @@ -687,10 +633,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_time_in_queue_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])\n/\nrate(executor_time_in_queue_seconds_count{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -723,10 +666,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values({__name__=~\"jvm.*\"}, job)", "hide": 0, "includeAll": true, @@ -757,10 +697,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(executor_threads_min{job=~\"$Application\"},instance)", "hide": 0, "includeAll": true, @@ -791,10 +728,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(executor_threads_min{job=~\"$Application\",instance=~\"$Instance\"},name)", "hide": 0, "includeAll": true, @@ -880,10 +814,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(executor_threads_min{job=~\"$Application\",instance=~\"$Instance\"},type)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Cache_Metrics.json b/deployment/operations/grafana-dashboards/Cache_Metrics.json index 4cae8324023..5eb25da45df 100644 --- a/deployment/operations/grafana-dashboards/Cache_Metrics.json +++ b/deployment/operations/grafana-dashboards/Cache_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Cluster_traffic.json b/deployment/operations/grafana-dashboards/Cluster_traffic.json index a837a2ad0b1..8723f3e16b0 100644 --- a/deployment/operations/grafana-dashboards/Cluster_traffic.json +++ b/deployment/operations/grafana-dashboards/Cluster_traffic.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -680,10 +680,7 @@ }, { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -696,10 +693,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -779,10 +773,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(enforcement_seconds_count{segment=\"overall\",channel!=\"live\"}[5m]) > 0) by (category, resource, outcome)", "format": "time_series", @@ -792,10 +783,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(enforcement_seconds_count{segment=\"overall\",channel=\"live\"}[5m]) > 0) by (category, resource, outcome)", "format": "time_series", @@ -810,10 +798,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -890,10 +875,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "avg((idelta(enforcement_seconds_sum{segment=\"overall\",channel!=\"live\"}[5m]) / idelta(enforcement_seconds_count{segment=\"overall\",channel!=\"live\"}[5m])) > 0) by (category, resource, outcome)", "format": "time_series", @@ -903,10 +885,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "avg((idelta(enforcement_seconds_sum{segment=\"overall\",channel=\"live\"}[5m]) / idelta(enforcement_seconds_count{segment=\"overall\",channel=\"live\"}[5m])) > 0) by (category, resource, outcome)", "format": "time_series", diff --git a/deployment/operations/grafana-dashboards/Connectivity_ACKS.json b/deployment/operations/grafana-dashboards/Connectivity_ACKS.json index b47c75f32a2..8ac2860b141 100644 --- a/deployment/operations/grafana-dashboards/Connectivity_ACKS.json +++ b/deployment/operations/grafana-dashboards/Connectivity_ACKS.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Connectivity_Metrics.json b/deployment/operations/grafana-dashboards/Connectivity_Metrics.json index 33822617a26..2a769a70426 100644 --- a/deployment/operations/grafana-dashboards/Connectivity_Metrics.json +++ b/deployment/operations/grafana-dashboards/Connectivity_Metrics.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -30,10 +27,7 @@ "panels": [ { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -50,10 +44,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -102,10 +93,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "sum(connection_client{instance=~\"$Host\", type=~\"$ConnectionType\", id=~\"$Connection\"}) by (type)", "format": "time_series", "instant": false, @@ -149,10 +137,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -201,10 +186,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "sum(connection_client{instance=~\"$Host\", type=~\"$ConnectionType\", id=~\"$Connection\"}) by (instance, type)", "format": "time_series", "instant": false, @@ -248,10 +230,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Connections which delivered metrics in the last 7 days", "fieldConfig": { "defaults": { @@ -301,10 +280,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "count by (ditto_connection_type) (count by (instance, ditto_connection_id, ditto_connection_type) (increase(connectivity_message_mapping_seconds_count{instance=~\"$Host\", ditto_connection_type=~\"$ConnectionType\", ditto_connection_id=~\"$Connection\"}[5m]) > 0))", "format": "time_series", "hide": false, @@ -349,10 +325,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Connections which delivered metrics in the last 7 days", "fieldConfig": { "defaults": { @@ -402,10 +375,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "count by (instance, ditto_connection_type) (count by (instance, ditto_connection_id, ditto_connection_type) (increase (connectivity_message_mapping_seconds_count{instance=~\"$Host\", ditto_connection_type=~\"$ConnectionType\", ditto_connection_id=~\"$Connection\"}[5m]) > 0))", "format": "time_series", "hide": false, @@ -447,10 +417,7 @@ }, { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -467,10 +434,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -566,10 +530,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "This number reflects the number of messages that resulted out of payload mappings per second.", "fieldConfig": { "defaults": { @@ -666,10 +627,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -766,10 +724,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -871,10 +826,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1006,10 +958,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1056,10 +1005,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": true, "expr": "count(rate(connection_messages_total{instance=~\"$Host\", id=~\"$Connection\", type=~\"$ConnectionType\", category=\"throttled\", direction=\"inbound\"}[5m])>0) by (type, id)", @@ -1071,10 +1017,7 @@ "refId": "B" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "exemplar": true, "expr": "count(rate(connection_messages_total{category=\"throttled\", direction=\"inbound\"}[5m]) > 0) by (id)", "hide": true, @@ -1131,10 +1074,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1228,10 +1168,7 @@ }, { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -1248,10 +1185,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1348,10 +1282,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1447,10 +1378,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "This number reflects the number of messages that resulted out of payload mappings per second.", "fieldConfig": { "defaults": { @@ -1547,10 +1475,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1646,10 +1571,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1745,10 +1667,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1844,10 +1763,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -1962,10 +1878,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -2074,10 +1987,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "", "hide": 0, "includeAll": true, @@ -2108,10 +2018,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(connectivity_message_mapping_seconds_count,ditto_connection_id)", "hide": 0, "includeAll": true, @@ -2138,10 +2045,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(connectivity_message_mapping_seconds_count,ditto_connection_type)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Connectivity_live_status.json b/deployment/operations/grafana-dashboards/Connectivity_live_status.json index 52796004638..3b3f266c3d9 100644 --- a/deployment/operations/grafana-dashboards/Connectivity_live_status.json +++ b/deployment/operations/grafana-dashboards/Connectivity_live_status.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Container_Metrics.json b/deployment/operations/grafana-dashboards/Container_Metrics.json index a3949c79f01..832cd2d6a82 100644 --- a/deployment/operations/grafana-dashboards/Container_Metrics.json +++ b/deployment/operations/grafana-dashboards/Container_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/External_Metrics.json b/deployment/operations/grafana-dashboards/External_Metrics.json index a21553f3643..67b79c3f72d 100644 --- a/deployment/operations/grafana-dashboards/External_Metrics.json +++ b/deployment/operations/grafana-dashboards/External_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Gateway_Traces.json b/deployment/operations/grafana-dashboards/Gateway_Traces.json index 3de7a4cb297..a9350acf75d 100644 --- a/deployment/operations/grafana-dashboards/Gateway_Traces.json +++ b/deployment/operations/grafana-dashboards/Gateway_Traces.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/JVM_Metrics.json b/deployment/operations/grafana-dashboards/JVM_Metrics.json index 2c7fdfe5920..21a526f700e 100644 --- a/deployment/operations/grafana-dashboards/JVM_Metrics.json +++ b/deployment/operations/grafana-dashboards/JVM_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Kafka_Consumer_Metrics.json b/deployment/operations/grafana-dashboards/Kafka_Consumer_Metrics.json index ef6a63d89da..2d468fbf34d 100644 --- a/deployment/operations/grafana-dashboards/Kafka_Consumer_Metrics.json +++ b/deployment/operations/grafana-dashboards/Kafka_Consumer_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Kubernetes_Metrics.json b/deployment/operations/grafana-dashboards/Kubernetes_Metrics.json index 72c17b5d1af..ddecf3ff22d 100644 --- a/deployment/operations/grafana-dashboards/Kubernetes_Metrics.json +++ b/deployment/operations/grafana-dashboards/Kubernetes_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Load_Test.json b/deployment/operations/grafana-dashboards/Load_Test.json index 514a65c2095..6c0544e7301 100644 --- a/deployment/operations/grafana-dashboards/Load_Test.json +++ b/deployment/operations/grafana-dashboards/Load_Test.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -29,10 +26,7 @@ "panels": [ { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -45,10 +39,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -132,10 +123,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -210,10 +198,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(enforcement_seconds_count{segment=\"overall\",channel!=\"live\"}[5m]) > 0) by (category, resource, outcome)", "format": "time_series", @@ -223,10 +208,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(enforcement_seconds_count{segment=\"overall\",channel=\"live\"}[5m]) > 0) by (category, resource, outcome)", "format": "time_series", @@ -242,10 +224,7 @@ }, { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -258,10 +237,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -354,10 +330,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -444,10 +417,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Positive: inbound messages\nNegative: outbound messages", "fieldConfig": { "defaults": { diff --git a/deployment/operations/grafana-dashboards/Persistence_Entities.json b/deployment/operations/grafana-dashboards/Persistence_Entities.json index a89f0319c5f..f69715730da 100644 --- a/deployment/operations/grafana-dashboards/Persistence_Entities.json +++ b/deployment/operations/grafana-dashboards/Persistence_Entities.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Pub_Sub.json b/deployment/operations/grafana-dashboards/Pub_Sub.json index 49135cb536c..a88ae765a2e 100644 --- a/deployment/operations/grafana-dashboards/Pub_Sub.json +++ b/deployment/operations/grafana-dashboards/Pub_Sub.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -33,10 +30,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -120,10 +114,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -222,10 +213,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -305,10 +293,7 @@ } }, { - "datasource": { - "type": "elasticsearch", - "uid": "PAE1B8C8635429669" - }, + "datasource": "elasticsearch", "fieldConfig": { "defaults": { "color": { @@ -555,10 +540,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -645,10 +627,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fill": 1, "fillGradient": 0, @@ -770,10 +749,7 @@ "noDataState": "no_data", "notifications": [] }, - "datasource": { - "type": "elasticsearch", - "uid": "PAE1B8C8635429669" - }, + "datasource": "elasticsearch", "fieldConfig": { "defaults": { "color": { @@ -882,10 +858,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -981,10 +954,7 @@ "templating": { "list": [ { - "datasource": { - "type": "elasticsearch", - "uid": "PAE1B8C8635429669" - }, + "datasource": "elasticsearch", "filters": [], "hide": 0, "name": "Filters", diff --git a/deployment/operations/grafana-dashboards/Signal_processing.json b/deployment/operations/grafana-dashboards/Signal_processing.json index 3a4e8f9ae20..d02ef2c1fe2 100644 --- a/deployment/operations/grafana-dashboards/Signal_processing.json +++ b/deployment/operations/grafana-dashboards/Signal_processing.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -42,10 +39,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -135,10 +129,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "sum(irate(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"overall\",outcome=~\"$Outcome\",channel!=\"live\"}[$__rate_interval])) by (resource, category, outcome)", @@ -152,10 +143,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -248,10 +236,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "sum(irate(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"overall\",channel=\"live\"}[$__rate_interval])) by (resource, category, outcome)", @@ -266,10 +251,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -341,10 +323,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"overall\",channel!=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"overall\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -358,10 +337,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -433,10 +409,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"overall\",channel=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"overall\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -464,10 +437,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -539,10 +509,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(pre_enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",channel!=\"live\"}[$__rate_interval]) / idelta(pre_enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -556,10 +523,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -631,10 +595,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(pre_enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",channel=\"live\"}[$__rate_interval]) / idelta(pre_enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -662,10 +623,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -737,10 +695,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"enf\",channel!=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"enf\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -754,10 +709,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -829,10 +781,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",segment=\"enf\",channel=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"enf\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -857,10 +806,7 @@ "id": 17, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -971,10 +917,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"process\",channel!=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"process\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -988,10 +931,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1063,10 +1003,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"process\",channel=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"process\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -1095,10 +1032,7 @@ "id": 21, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1170,10 +1104,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"resp_filter\",channel!=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"resp_filter\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -1187,10 +1118,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1262,10 +1190,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"resp_filter\",channel=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"resp_filter\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -1297,10 +1222,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(enforcement_seconds_count, job)", "hide": 0, "includeAll": true, @@ -1324,10 +1246,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(enforcement_seconds_count{job=\"$Application\"}, instance)", "hide": 0, "includeAll": true, @@ -1351,10 +1270,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(enforcement_seconds_count, outcome)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Sudo_command_count.json b/deployment/operations/grafana-dashboards/Sudo_command_count.json index a25d5992242..587476ea225 100644 --- a/deployment/operations/grafana-dashboards/Sudo_command_count.json +++ b/deployment/operations/grafana-dashboards/Sudo_command_count.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -29,10 +26,7 @@ "liveNow": false, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -105,10 +99,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(sudo_commands_total{job=~\"$Application\",instance=~\"$Instance\"}[5m])) by (type)", "legendFormat": "{{type}}", @@ -120,10 +111,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -189,10 +177,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "sum(sudo_commands_total{job=~\"$Application\",instance=~\"$Instance\"}) by (type)", @@ -241,10 +226,7 @@ "type": "table" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -290,10 +272,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "sum(sudo_commands_total{job=~\"$Application\",instance=~\"$Instance\"}) by (type)", @@ -334,10 +313,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(sudo_commands_total, job)", "hide": 0, "includeAll": true, @@ -361,10 +337,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(sudo_commands_total, instance)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Things-Wildcard-Search_Performance_Metrics.json b/deployment/operations/grafana-dashboards/Things-Wildcard-Search_Performance_Metrics.json index 29353fd8d7a..6637c5084a0 100644 --- a/deployment/operations/grafana-dashboards/Things-Wildcard-Search_Performance_Metrics.json +++ b/deployment/operations/grafana-dashboards/Things-Wildcard-Search_Performance_Metrics.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -35,10 +32,7 @@ "cacheTimeout": "0", "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Positive Y axis: stacked search queries\nNegative Y axis: stacked search counts", "editable": true, "error": false, @@ -143,10 +137,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editable": true, "error": false, "fieldConfig": { @@ -242,10 +233,7 @@ "cacheTimeout": "0", "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editable": true, "error": false, "fieldConfig": { @@ -343,10 +331,7 @@ "cacheTimeout": "0", "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Percentiles are capped to max. 10s", "editable": true, "error": false, @@ -475,10 +460,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -568,10 +550,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -675,10 +654,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -764,10 +740,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -857,10 +830,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -990,10 +960,7 @@ } }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -1087,10 +1054,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -1180,10 +1144,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -1273,10 +1234,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -1383,10 +1341,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-datasources/datasource.yaml b/deployment/operations/grafana-datasources/datasource.yaml new file mode 100644 index 00000000000..14f5b356dd7 --- /dev/null +++ b/deployment/operations/grafana-datasources/datasource.yaml @@ -0,0 +1,39 @@ +# config file version +apiVersion: 1 + +# list of datasources to insert/update depending +# whats available in the database +datasources: + # + # Prometheus datasource + # +- name: prometheus + # datasource type. Required + type: prometheus + # access mode. direct or proxy. Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://prometheus:9090 + # allow users to edit datasources from the UI. + editable: false + +- name: elasticsearch + # datasource type. Required + type: elasticsearch + # access mode. direct or proxy. Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://elasticsearch:9200 + # allow users to edit datasources from the UI. + editable: false + database: "[logstash-]YYYY.MM.DD" + # fields that will be converted to json and stored in json_data + jsonData: + interval: Daily + timeField: "@timestamp" + esVersion: '8.0.0' + logMessageField: message diff --git a/deployment/operations/prometheus/prometheus.yml b/deployment/operations/prometheus/prometheus.yml new file mode 100644 index 00000000000..a5a50a5676f --- /dev/null +++ b/deployment/operations/prometheus/prometheus.yml @@ -0,0 +1,42 @@ +global: + scrape_interval: 30s + scrape_timeout: 10s + evaluation_interval: 30s + +scrape_configs: + # Scrape prometheus itself. + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # Scrape grafana. + - job_name: 'grafana' + static_configs: + - targets: ['grafana:3000'] + + # Scrape Ditto. + - job_name: 'ditto' + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_path ] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: ${1}:${2} + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: instance + - source_labels: [__meta_kubernetes_pod_container_name] + action: replace + target_label: job diff --git a/documentation/README.md b/documentation/README.md index 706f130e04b..eb0cc54181d 100755 --- a/documentation/README.md +++ b/documentation/README.md @@ -1,6 +1,6 @@ ## Eclipse Ditto :: Documentation -This folder contains the documentation and [static website of Eclipse Ditto](https://www.eclipse.org/ditto/). +This folder contains the documentation and [static website of Eclipse Ditto](https://www.eclipse.dev/ditto/). The documentation is based on [Jekyll](https://jekyllrb.com) and the fabulous [Jekyll Documentation Theme 6.0](http://idratherbewriting.com/documentation-theme-jekyll/). @@ -49,7 +49,7 @@ bundle exec jekyll serve --verbose --unpublished Validate that the HTML does not contain dead links, etc.: ```bash -htmlproofer --assume-extension --allow-hash-href --disable-external --enforce-https=false --ignore-urls "/http-api-doc.html.*/" src/main/resources/_site/ +htmlproofer --assume-extension --allow-hash-href --disable-external --ignore-urls "/http-api-doc.html.*/","http://localhost:4000/feed.xml","http://www.ontology-of-units-of-measure.org/page/om-2" src/main/resources/_site/ ``` #### Alternative 2: use Maven (UNIX) diff --git a/documentation/src/main/resources/Gemfile b/documentation/src/main/resources/Gemfile index 05068a53e5c..957b8ed6e5f 100644 --- a/documentation/src/main/resources/Gemfile +++ b/documentation/src/main/resources/Gemfile @@ -8,7 +8,7 @@ source "https://rubygems.org" # # This will help ensure the proper Jekyll version is running. # Happy Jekylling! -gem "jekyll", "~> 4.2.2" +gem "jekyll", "~> 4.3.2" # This is the default theme for new Jekyll sites. You may change this to anything you like. diff --git a/documentation/src/main/resources/Gemfile.lock b/documentation/src/main/resources/Gemfile.lock index 2a43b377843..d4a9af012af 100644 --- a/documentation/src/main/resources/Gemfile.lock +++ b/documentation/src/main/resources/Gemfile.lock @@ -1,36 +1,38 @@ GEM remote: https://rubygems.org/ specs: - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) colorator (1.1.0) - concurrent-ruby (1.1.10) + concurrent-ruby (1.2.0) em-websocket (0.5.3) eventmachine (>= 0.12.9) http_parser.rb (~> 0) eventmachine (1.2.7) ffi (1.15.5) forwardable-extended (2.6.0) + google-protobuf (3.22.0-arm64-darwin) http_parser.rb (0.8.0) i18n (1.12.0) concurrent-ruby (~> 1.0) - jekyll (4.2.2) + jekyll (4.3.2) addressable (~> 2.4) colorator (~> 1.0) em-websocket (~> 0.5) i18n (~> 1.0) - jekyll-sass-converter (~> 2.0) + jekyll-sass-converter (>= 2.0, < 4.0) jekyll-watch (~> 2.0) - kramdown (~> 2.3) + kramdown (~> 2.3, >= 2.3.1) kramdown-parser-gfm (~> 1.0) liquid (~> 4.0) - mercenary (~> 0.4.0) + mercenary (>= 0.3.6, < 0.5) pathutil (~> 0.9) - rouge (~> 3.0) + rouge (>= 3.0, < 5.0) safe_yaml (~> 1.0) - terminal-table (~> 2.0) - jekyll-sass-converter (2.2.0) - sassc (> 2.0.1, < 3.0) + terminal-table (>= 1.8, < 4.0) + webrick (~> 1.7) + jekyll-sass-converter (3.0.0) + sass-embedded (~> 1.54) jekyll-sitemap (1.4.0) jekyll (>= 3.7, < 5.0) jekyll-watch (2.2.1) @@ -39,32 +41,33 @@ GEM rexml kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) - liquid (4.0.3) - listen (3.7.1) + liquid (4.0.4) + listen (3.8.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.4.0) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (4.0.7) - rb-fsevent (0.11.1) + public_suffix (5.0.1) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) rexml (3.2.5) - rouge (3.29.0) + rouge (4.1.0) safe_yaml (1.0.5) - sassc (2.4.0) - ffi (~> 1.9) - terminal-table (2.0.0) - unicode-display_width (~> 1.1, >= 1.1.1) - unicode-display_width (1.8.0) + sass-embedded (1.58.3-arm64-darwin) + google-protobuf (~> 3.21) + terminal-table (3.0.2) + unicode-display_width (>= 1.1.1, < 3) + unicode-display_width (2.4.2) webrick (1.7.0) PLATFORMS arm64-darwin-21 + arm64-darwin-22 DEPENDENCIES - jekyll (~> 4.2.2) + jekyll (~> 4.3.2) jekyll-sitemap (~> 1.4.0) tzinfo-data webrick (~> 1.7.0) diff --git a/documentation/src/main/resources/_config.yml b/documentation/src/main/resources/_config.yml index 517bf8d7893..8d6b64a8c08 100644 --- a/documentation/src/main/resources/_config.yml +++ b/documentation/src/main/resources/_config.yml @@ -114,17 +114,13 @@ plugins: docVersions: - label: "development" basePath: "" + - label: "3.3" + basePath: "3.3" + - label: "3.2" + basePath: "3.2" - label: "3.1" basePath: "3.1" - label: "3.0" basePath: "3.0" - label: "2.4" basePath: "2.4" - - label: "2.3" - basePath: "2.3" - - label: "2.2" - basePath: "2.2" - - label: "2.1" - basePath: "2.1" - - label: "2.0" - basePath: "2.0" diff --git a/documentation/src/main/resources/_data/authors.yml b/documentation/src/main/resources/_data/authors.yml index 93333c8d446..1e286541d99 100644 --- a/documentation/src/main/resources/_data/authors.yml +++ b/documentation/src/main/resources/_data/authors.yml @@ -1,6 +1,6 @@ thomas_jaeckle: name: Thomas Jäckle - email: thomas.jaeckle@bosch.io + email: thomas.jaeckle@beyonnex.io web: https://github.com/thjaeckle florian_fendt: @@ -10,7 +10,7 @@ florian_fendt: juergen_fickel: name: Jürgen Fickel - email: juergen.fickel@bosch.io + email: eclipse-foundation@retujo.de web: https://github.com/jufickel-b philipp_michalski: diff --git a/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml b/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml index 4ac16530a28..48c7c3de88d 100644 --- a/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml +++ b/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml @@ -23,29 +23,50 @@ entries: - title: Release Notes output: web folderitems: - - title: 3.1.1 - url: /release_notes_311.html + - title: 3.3.4 + url: /release_notes_334.html output: web - - title: 3.1.0 - url: /release_notes_310.html + - title: 3.3.3 + url: /release_notes_333.html output: web - - title: 3.0.0 - url: /release_notes_300.html + - title: 3.3.2 + url: /release_notes_332.html output: web - - title: 2.4.2 - url: /release_notes_242.html + - title: 3.3.0 + url: /release_notes_330.html output: web - - title: 2.4.1 - url: /release_notes_241.html + - title: 3.2.1 + url: /release_notes_321.html output: web - - title: 2.4.0 - url: /release_notes_240.html + - title: 3.2.0 + url: /release_notes_320.html output: web subfolders: - title: Archive output: web subfolderitems: + - title: 3.1.2 + url: /release_notes_312.html + output: web + - title: 3.1.1 + url: /release_notes_311.html + output: web + - title: 3.1.0 + url: /release_notes_310.html + output: web + - title: 3.0.0 + url: /release_notes_300.html + output: web + - title: 2.4.2 + url: /release_notes_242.html + output: web + - title: 2.4.1 + url: /release_notes_241.html + output: web + - title: 2.4.0 + url: /release_notes_240.html + output: web - title: 2.3.2 url: /release_notes_232.html output: web @@ -263,6 +284,9 @@ entries: - title: Search url: /basic-search.html output: web + - title: History capabilities + url: /basic-history.html + output: web - title: Acknowledgements / QoS url: /basic-acknowledgements.html output: web @@ -492,6 +516,10 @@ entries: url: /protocol-specification-connections-announcement.html output: web + - title: Streaming subscriptions (history) + url: /protocol-specification-streaming-subscription.html + output: web + - title: Bindings url: /protocol-bindings.html output: web diff --git a/documentation/src/main/resources/_data/tags.yml b/documentation/src/main/resources/_data/tags.yml index 84db28d555c..72d45c03e79 100644 --- a/documentation/src/main/resources/_data/tags.yml +++ b/documentation/src/main/resources/_data/tags.yml @@ -11,6 +11,7 @@ allowed-tags: - model - signal - http + - history - search - protocol - connectivity diff --git a/documentation/src/main/resources/_data/topnav.yml b/documentation/src/main/resources/_data/topnav.yml index 4bf938138ec..7a473874854 100644 --- a/documentation/src/main/resources/_data/topnav.yml +++ b/documentation/src/main/resources/_data/topnav.yml @@ -13,15 +13,15 @@ topnav: - title: image: GitHub-Mark-Light-32px.png alt: Sources at GitHub - external_url: https://github.com/eclipse/ditto + external_url: https://github.com/eclipse-ditto/ditto - title: SDKs image: GitHub-Mark-Light-32px.png alt: SDK sources at GitHub - external_url: https://github.com/eclipse/ditto-clients + external_url: https://github.com/eclipse-ditto/ditto-clients - title: examples image: GitHub-Mark-Light-32px.png alt: Example sources at GitHub - external_url: https://github.com/eclipse/ditto-examples + external_url: https://github.com/eclipse-ditto/ditto-examples #Topnav dropdowns topnav_dropdowns: diff --git a/documentation/src/main/resources/_includes/head.html b/documentation/src/main/resources/_includes/head.html index 8e92a1e37f9..f85c61837f4 100644 --- a/documentation/src/main/resources/_includes/head.html +++ b/documentation/src/main/resources/_includes/head.html @@ -6,17 +6,17 @@ {% if page.title %} {% if page.layout == "post" %} {{ page.title }} • {{ site.topnav_title }}{% else %} {{ page.title }} • {{ site.short_title }}{% endif %}{% else %}{{ site.site_title }}{% endif %} - + - + - - - + + + @@ -24,8 +24,8 @@ { "@context": "http://schema.org", "@type": "Organization", - "url": "https://www.eclipse.org/ditto/", - "logo": "https://www.eclipse.org/ditto/images/ditto.svg" + "url": "https://www.eclipse.dev/ditto/", + "logo": "https://www.eclipse.dev/ditto/images/ditto.svg" } @@ -36,5 +36,5 @@ - - + + diff --git a/documentation/src/main/resources/_includes/topnav.html b/documentation/src/main/resources/_includes/topnav.html index b97d3cfb6a4..d17e798b338 100644 --- a/documentation/src/main/resources/_includes/topnav.html +++ b/documentation/src/main/resources/_includes/topnav.html @@ -72,7 +72,7 @@
    - + - + + + + + + + + + + + + diff --git a/documentation/src/main/resources/wot/ditto-extension.html b/documentation/src/main/resources/wot/ditto-extension.html index 83809d8f1a5..71297c22264 100644 --- a/documentation/src/main/resources/wot/ditto-extension.html +++ b/documentation/src/main/resources/wot/ditto-extension.html @@ -61,7 +61,7 @@

    - WoT Extension Ontology

    1. Introduction

    -

    This ontology provides additional Eclipse Ditto™ specific context for W3C Web of Things TDs and TMs.

    +

    This ontology provides additional Eclipse Ditto™ specific context for W3C Web of Things TDs and TMs.

    2. Formats

    diff --git a/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/AskWithRetryCommandForwarder.java b/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/AskWithRetryCommandForwarder.java index d5a1ba1492c..368cc93bee3 100644 --- a/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/AskWithRetryCommandForwarder.java +++ b/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/AskWithRetryCommandForwarder.java @@ -208,7 +208,7 @@ private DittoRuntimeException reportError(final Command command, : throwable; final var dre = DittoRuntimeException.asDittoRuntimeException( error, t -> reportUnexpectedError(command, t)); - LOGGER.info(" - {}: {}", dre.getClass().getSimpleName(), dre.getMessage()); + LOGGER.withCorrelationId(command).info("{}: {}", dre.getClass().getSimpleName(), dre.getMessage()); return dre; } diff --git a/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/EdgeCommandForwarderActor.java b/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/EdgeCommandForwarderActor.java index 6db6073e0e1..8fc2786b364 100644 --- a/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/EdgeCommandForwarderActor.java +++ b/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/EdgeCommandForwarderActor.java @@ -22,11 +22,13 @@ import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; import org.eclipse.ditto.base.model.signals.events.Event; import org.eclipse.ditto.base.service.signaltransformer.SignalTransformer; import org.eclipse.ditto.base.service.signaltransformer.SignalTransformers; import org.eclipse.ditto.connectivity.api.ConnectivityMessagingConstants; import org.eclipse.ditto.connectivity.api.commands.sudo.ConnectivitySudoCommand; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveAllConnectionIds; import org.eclipse.ditto.internal.utils.akka.logging.DittoDiagnosticLoggingAdapter; @@ -37,8 +39,10 @@ import org.eclipse.ditto.internal.utils.config.ScopedConfig; import org.eclipse.ditto.messages.model.signals.commands.MessageCommand; import org.eclipse.ditto.messages.model.signals.commands.MessageCommandResponse; +import org.eclipse.ditto.policies.model.PolicyConstants; import org.eclipse.ditto.policies.model.signals.commands.PolicyCommand; import org.eclipse.ditto.things.api.commands.sudo.SudoRetrieveThings; +import org.eclipse.ditto.things.model.ThingConstants; import org.eclipse.ditto.things.model.signals.commands.ThingCommand; import org.eclipse.ditto.things.model.signals.commands.ThingCommandResponse; import org.eclipse.ditto.things.model.signals.commands.query.RetrieveThings; @@ -146,6 +150,18 @@ public Receive createReceive() { .match(ConnectivitySudoCommand.class, this::forwardToConnectivity) .match(ThingSearchCommand.class, this::forwardToThingSearch) .match(ThingSearchSudoCommand.class, this::forwardToThingSearch) + .match(StreamingSubscriptionCommand.class, + src -> src.getEntityType().equals(ThingConstants.ENTITY_TYPE), + this::forwardToThings + ) + .match(StreamingSubscriptionCommand.class, + src -> src.getEntityType().equals(PolicyConstants.ENTITY_TYPE), + this::forwardToPolicies + ) + .match(StreamingSubscriptionCommand.class, + src -> src.getEntityType().equals(ConnectivityConstants.ENTITY_TYPE), + this::forwardToConnectivity + ) .match(Signal.class, this::handleUnknownSignal) .matchAny(m -> log.warning("Got unknown message: {}", m)) .build(); @@ -220,22 +236,23 @@ private void forwardToThingsAggregatorProxy(final Command command) { () -> signalTransformationCs.thenAccept(transformed -> aggregatorProxyActor.tell(transformed, sender))); } - private void forwardToPolicies(final PolicyCommand policyCommand) { + private void forwardToPolicies(final Signal policySignal) { final ActorRef sender = getSender(); - final CompletionStage> signalTransformationCs = applySignalTransformation(policyCommand, sender); - scheduleTask(policyCommand, () -> signalTransformationCs - .thenAccept(transformed -> { - final PolicyCommand transformedPolicyCommand = (PolicyCommand) transformed; - log.withCorrelationId(transformedPolicyCommand) + final CompletionStage> signalTransformationCs = applySignalTransformation(policySignal, sender); + scheduleTask(policySignal, () -> signalTransformationCs + .thenAccept(transformedSignal -> { + log.withCorrelationId(transformedSignal) .info("Forwarding policy command with ID <{}> and type <{}> to 'policies' shard region", - transformedPolicyCommand.getEntityId(), transformedPolicyCommand.getType()); + transformedSignal instanceof WithEntityId withEntityId ? withEntityId.getEntityId() : + null, + transformedSignal.getType()); - if (isIdempotent(transformedPolicyCommand)) { - askWithRetryCommandForwarder.forwardCommand(transformedPolicyCommand, + if (transformedSignal instanceof Command transformedCommand && isIdempotent(transformedCommand)) { + askWithRetryCommandForwarder.forwardCommand(transformedCommand, shardRegions.policies(), sender); } else { - shardRegions.policies().tell(transformedPolicyCommand, sender); + shardRegions.policies().tell(transformedSignal, sender); } })); } diff --git a/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionActor.java b/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionActor.java new file mode 100644 index 00000000000..5138fc8ae02 --- /dev/null +++ b/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionActor.java @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.edge.service.streaming; + +import java.time.Duration; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.commands.exceptions.StreamingSubscriptionProtocolErrorException; +import org.eclipse.ditto.base.model.signals.commands.exceptions.StreamingSubscriptionTimeoutException; +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionCreated; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionHasNext; +import org.eclipse.ditto.internal.utils.akka.actors.AbstractActorWithStashWithTimers; +import org.eclipse.ditto.internal.utils.akka.logging.DittoDiagnosticLoggingAdapter; +import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory; +import org.eclipse.ditto.json.JsonValue; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +import akka.actor.ActorRef; +import akka.actor.PoisonPill; +import akka.actor.Props; +import akka.actor.ReceiveTimeout; +import akka.japi.pf.ReceiveBuilder; + +/** + * Actor that translates streaming subscription commands into stream operations and stream signals into streaming + * subscription events. + */ +public final class StreamingSubscriptionActor extends AbstractActorWithStashWithTimers { + + /** + * Live on as zombie for a while to prevent timeout at client side + */ + private static final Duration ZOMBIE_LIFETIME = Duration.ofSeconds(10L); + + private final DittoDiagnosticLoggingAdapter log = DittoLoggerFactory.getDiagnosticLoggingAdapter(this); + + private final EntityId entityId; + private Subscription subscription; + private ActorRef sender; + private DittoHeaders dittoHeaders; + + StreamingSubscriptionActor(final Duration idleTimeout, + final EntityId entityId, + final ActorRef sender, + final DittoHeaders dittoHeaders) { + this.entityId = entityId; + this.sender = sender; + this.dittoHeaders = dittoHeaders; + getContext().setReceiveTimeout(idleTimeout); + } + + /** + * Create Props object for the StreamingSubscriptionActor. + * + * @param idleTimeout maximum lifetime while idling + * @param entityId the entity ID for which the streaming subscription is created. + * @param sender sender of the command that created this actor. + * @param dittoHeaders headers of the command that created this actor. + * @return Props for this actor. + */ + public static Props props(final Duration idleTimeout, final EntityId entityId, final ActorRef sender, + final DittoHeaders dittoHeaders) { + return Props.create(StreamingSubscriptionActor.class, idleTimeout, entityId, sender, dittoHeaders); + } + + /** + * Wrap a subscription actor as a reactive stream subscriber. + * + * @param streamingSubscriptionActor reference to the subscription actor. + * @param entityId the entity ID for which the subscriber is created. + * @return the actor presented as a reactive stream subscriber. + */ + public static Subscriber asSubscriber(final ActorRef streamingSubscriptionActor, + final EntityId entityId) { + return new StreamingSubscriberOps(streamingSubscriptionActor, entityId); + } + + @Override + public void postStop() { + if (subscription != null) { + subscription.cancel(); + } + } + + @Override + public Receive createReceive() { + return ReceiveBuilder.create() + .match(RequestFromStreamingSubscription.class, this::requestSubscription) + .match(CancelStreamingSubscription.class, this::cancelSubscription) + .match(StreamingSubscriptionHasNext.class, this::subscriptionHasNext) + .match(StreamingSubscriptionComplete.class, this::subscriptionComplete) + .match(StreamingSubscriptionFailed.class, this::subscriptionFailed) + .match(Subscription.class, this::onSubscribe) + .matchEquals(ReceiveTimeout.getInstance(), this::idleTimeout) + .matchAny(m -> log.warning("Unknown message: <{}>", m)) + .build(); + } + + private Receive createZombieBehavior() { + return ReceiveBuilder.create() + .match(RequestFromStreamingSubscription.class, requestSubscription -> { + log.withCorrelationId(requestSubscription) + .info("Rejecting RequestSubscription[demand={}] as zombie", + requestSubscription.getDemand()); + final String errorMessage = + "This subscription is considered cancelled. No more messages are processed."; + final StreamingSubscriptionFailed subscriptionFailed = StreamingSubscriptionFailed.of( + getSubscriptionId(), + entityId, + StreamingSubscriptionProtocolErrorException.newBuilder() + .message(errorMessage) + .build(), + requestSubscription.getDittoHeaders() + ); + getSender().tell(subscriptionFailed, ActorRef.noSender()); + }) + .matchAny(message -> log.debug("Ignoring as zombie: <{}>", message)) + .build(); + } + + private void idleTimeout(final ReceiveTimeout receiveTimeout) { + // usually a user error + log.info("Stopping due to idle timeout"); + getContext().cancelReceiveTimeout(); + final String subscriptionId = getSubscriptionId(); + final StreamingSubscriptionTimeoutException error = StreamingSubscriptionTimeoutException + .of(subscriptionId, dittoHeaders); + final StreamingSubscriptionFailed subscriptionFailed = StreamingSubscriptionFailed + .of(subscriptionId, entityId, error, dittoHeaders); + if (subscription == null) { + sender.tell(getSubscriptionCreated(), ActorRef.noSender()); + } + sender.tell(subscriptionFailed, ActorRef.noSender()); + becomeZombie(); + } + + private void onSubscribe(final Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + } else { + this.subscription = subscription; + sender.tell(getSubscriptionCreated(), ActorRef.noSender()); + unstashAll(); + } + } + + private StreamingSubscriptionCreated getSubscriptionCreated() { + return StreamingSubscriptionCreated.of(getSubscriptionId(), entityId, dittoHeaders); + } + + private void setSenderAndDittoHeaders(final StreamingSubscriptionCommand command) { + sender = getSender(); + dittoHeaders = command.getDittoHeaders(); + } + + private void requestSubscription(final RequestFromStreamingSubscription requestFromStreamingSubscription) { + if (subscription == null) { + log.withCorrelationId(requestFromStreamingSubscription).debug("Stashing <{}>", requestFromStreamingSubscription); + stash(); + } else { + log.withCorrelationId(requestFromStreamingSubscription).debug("Processing <{}>", requestFromStreamingSubscription); + setSenderAndDittoHeaders(requestFromStreamingSubscription); + subscription.request(requestFromStreamingSubscription.getDemand()); + } + } + + private void cancelSubscription(final CancelStreamingSubscription cancelStreamingSubscription) { + if (subscription == null) { + log.withCorrelationId(cancelStreamingSubscription).info("Stashing <{}>", cancelStreamingSubscription); + stash(); + } else { + log.withCorrelationId(cancelStreamingSubscription).info("Processing <{}>", cancelStreamingSubscription); + setSenderAndDittoHeaders(cancelStreamingSubscription); + subscription.cancel(); + becomeZombie(); + } + } + + private void subscriptionHasNext(final StreamingSubscriptionHasNext event) { + log.debug("Forwarding {}", event); + sender.tell(event.setDittoHeaders(dittoHeaders), ActorRef.noSender()); + } + + private void subscriptionComplete(final StreamingSubscriptionComplete event) { + // just in case: if error overtakes subscription, then there *will* be a subscription. + if (subscription == null) { + log.withCorrelationId(event).debug("Stashing <{}>", event); + stash(); + } else { + log.info("{}", event); + sender.tell(event.setDittoHeaders(dittoHeaders), ActorRef.noSender()); + becomeZombie(); + } + } + + private void subscriptionFailed(final StreamingSubscriptionFailed event) { + // just in case: if error overtakes subscription, then there *will* be a subscription. + if (subscription == null) { + log.withCorrelationId(event).debug("Stashing <{}>", event); + stash(); + } else { + // log at INFO level because user errors may cause subscription failure. + log.withCorrelationId(event).info("{}", event); + sender.tell(event.setDittoHeaders(dittoHeaders), ActorRef.noSender()); + becomeZombie(); + } + } + + private void becomeZombie() { + getTimers().startSingleTimer(PoisonPill.getInstance(), PoisonPill.getInstance(), ZOMBIE_LIFETIME); + getContext().become(createZombieBehavior()); + } + + private String getSubscriptionId() { + return getSelf().path().name(); + } + + private static final class StreamingSubscriberOps implements Subscriber { + + private final ActorRef streamingSubscriptionActor; + private final String subscriptionId; + private final EntityId entityId; + + private StreamingSubscriberOps(final ActorRef streamingSubscriptionActor, final EntityId entityId) { + this.streamingSubscriptionActor = streamingSubscriptionActor; + subscriptionId = streamingSubscriptionActor.path().name(); + this.entityId = entityId; + } + + @Override + public void onSubscribe(final Subscription subscription) { + streamingSubscriptionActor.tell(subscription, ActorRef.noSender()); + } + + @Override + public void onNext(final JsonValue item) { + final StreamingSubscriptionHasNext event = StreamingSubscriptionHasNext + .of(subscriptionId, entityId, item, DittoHeaders.empty()); + streamingSubscriptionActor.tell(event, ActorRef.noSender()); + } + + @Override + public void onError(final Throwable t) { + final StreamingSubscriptionFailed event = + StreamingSubscriptionFailed.of(subscriptionId, + entityId, + DittoRuntimeException.asDittoRuntimeException(t, e -> { + if (e instanceof IllegalArgumentException) { + // incorrect protocol from the client side + return StreamingSubscriptionProtocolErrorException.of(e, DittoHeaders.empty()); + } else { + return DittoInternalErrorException.newBuilder().cause(e).build(); + } + }), + DittoHeaders.empty()); + streamingSubscriptionActor.tell(event, ActorRef.noSender()); + } + + @Override + public void onComplete() { + final StreamingSubscriptionComplete event = StreamingSubscriptionComplete.of(subscriptionId, entityId, + DittoHeaders.empty()); + streamingSubscriptionActor.tell(event, ActorRef.noSender()); + } + } +} diff --git a/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionManager.java b/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionManager.java new file mode 100644 index 00000000000..f21b6afdccb --- /dev/null +++ b/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionManager.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.edge.service.streaming; + +import java.time.Duration; +import java.util.Optional; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.json.Jsonifiable; +import org.eclipse.ditto.base.model.signals.FeatureToggle; +import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.base.model.signals.commands.exceptions.StreamingSubscriptionNotFoundException; +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.internal.utils.akka.logging.DittoDiagnosticLoggingAdapter; +import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory; +import org.eclipse.ditto.json.JsonValue; +import org.eclipse.ditto.protocol.ProtocolFactory; +import org.eclipse.ditto.protocol.adapter.DittoProtocolAdapter; +import org.reactivestreams.Subscriber; + +import akka.actor.AbstractActor; +import akka.actor.ActorRef; +import akka.actor.ActorSelection; +import akka.actor.Props; +import akka.japi.pf.ReceiveBuilder; +import akka.pattern.Patterns; +import akka.stream.Materializer; +import akka.stream.SourceRef; +import akka.stream.javadsl.Sink; +import akka.stream.javadsl.Source; + +/** + * Actor that manages streaming subscriptions for 1 websocket connection or 1 ConnectionPersistenceActor. + */ +public final class StreamingSubscriptionManager extends AbstractActor { + + /** + * Name of this actor. + */ + public static final String ACTOR_NAME = "streamingSubscriptionManager"; + + private static final DittoProtocolAdapter DITTO_PROTOCOL_ADAPTER = DittoProtocolAdapter.newInstance(); + private static final Duration COMMAND_FORWARDER_LOCAL_ASK_TIMEOUT = Duration.ofSeconds(15); + + private final DittoDiagnosticLoggingAdapter log = DittoLoggerFactory.getDiagnosticLoggingAdapter(this); + + private final Duration idleTimeout; + private final ActorSelection commandForwarder; + private final Materializer materializer; + + private int subscriptionIdCounter = 0; + + @SuppressWarnings("unused") + private StreamingSubscriptionManager(final Duration idleTimeout, + final ActorSelection commandForwarder, + final Materializer materializer) { + this.idleTimeout = idleTimeout; + this.commandForwarder = commandForwarder; + this.materializer = materializer; + } + + /** + * Create Props for a subscription manager. + * + * @param idleTimeout lifetime of an idle StreamingSubscriptionActor. + * @param commandForwarder recipient of streaming subscription commands. + * @param materializer materializer for the search streams. + * @return Props of the actor. + */ + public static Props props(final Duration idleTimeout, + final ActorSelection commandForwarder, + final Materializer materializer) { + + return Props.create(StreamingSubscriptionManager.class, idleTimeout, commandForwarder, materializer); + } + + @Override + public Receive createReceive() { + return ReceiveBuilder.create() + .match(RequestFromStreamingSubscription.class, this::requestSubscription) + .match(SubscribeForPersistedEvents.class, this::subscribeForPersistedEvents) + .match(CancelStreamingSubscription.class, this::cancelSubscription) + .build(); + } + + private void requestSubscription(final RequestFromStreamingSubscription requestFromStreamingSubscription) { + forwardToChild(requestFromStreamingSubscription.getSubscriptionId(), requestFromStreamingSubscription); + } + + private void cancelSubscription(final CancelStreamingSubscription cancelStreamingSubscription) { + forwardToChild(cancelStreamingSubscription.getSubscriptionId(), cancelStreamingSubscription); + } + + private void forwardToChild(final String streamingSubscriptionId, final StreamingSubscriptionCommand command) { + final Optional subscriptionActor = getContext().findChild(streamingSubscriptionId); + if (subscriptionActor.isPresent()) { + log.withCorrelationId(command).debug("Forwarding to child: <{}>", command); + subscriptionActor.get().tell(command, getSender()); + } else { + // most likely a user error + log.withCorrelationId(command) + .info("StreamingSubscriptionID not found, responding with StreamingSubscriptionFailed: <{}>", command); + final StreamingSubscriptionNotFoundException error = + StreamingSubscriptionNotFoundException.of(streamingSubscriptionId, command.getDittoHeaders()); + final StreamingSubscriptionFailed streamingSubscriptionFailed = + StreamingSubscriptionFailed.of(streamingSubscriptionId, command.getEntityId(), error, command.getDittoHeaders()); + getSender().tell(streamingSubscriptionFailed, ActorRef.noSender()); + } + } + + private void subscribeForPersistedEvents(final SubscribeForPersistedEvents subscribeForPersistedEvents) { + FeatureToggle.checkHistoricalApiAccessFeatureEnabled( + subscribeForPersistedEvents.getType(), subscribeForPersistedEvents.getDittoHeaders()); + + log.withCorrelationId(subscribeForPersistedEvents) + .info("Processing <{}>", subscribeForPersistedEvents); + final EntityId entityId = subscribeForPersistedEvents.getEntityId(); + final String subscriptionId = nextSubscriptionId(subscribeForPersistedEvents); + final Props props = StreamingSubscriptionActor.props(idleTimeout, entityId, getSender(), + subscribeForPersistedEvents.getDittoHeaders()); + final ActorRef subscriptionActor = getContext().actorOf(props, subscriptionId); + final Source itemSource = getPersistedEventsSource(subscribeForPersistedEvents); + connect(subscriptionActor, itemSource, entityId); + } + + private void connect(final ActorRef streamingSubscriptionActor, + final Source itemSource, + final EntityId entityId) { + final Subscriber subscriber = + StreamingSubscriptionActor.asSubscriber(streamingSubscriptionActor, entityId); + lazify(itemSource).runWith(Sink.fromSubscriber(subscriber), materializer); + } + + private Source getPersistedEventsSource(final SubscribeForPersistedEvents subscribe) { + + return Source.completionStageSource( + Patterns.ask(commandForwarder, subscribe, subscribe.getDittoHeaders() + .getTimeout() + .orElse(COMMAND_FORWARDER_LOCAL_ASK_TIMEOUT) + ) + .handle((response, throwable) -> { + if (response instanceof SourceRef sourceRef) { + return sourceRef.getSource() + .map(item -> { + if (item instanceof Signal signal) { + return ProtocolFactory.wrapAsJsonifiableAdaptable( + DITTO_PROTOCOL_ADAPTER.toAdaptable(signal) + ).toJson(); + } else if (item instanceof Jsonifiable jsonifiable) { + return jsonifiable.toJson(); + } else if (item instanceof JsonValue val) { + return val; + } else { + throw new IllegalStateException("Unexpected element!"); + } + }); + } else if (response instanceof DittoRuntimeException dittoRuntimeException) { + return Source.failed(dittoRuntimeException); + } else { + final var dittoRuntimeException = DittoRuntimeException + .asDittoRuntimeException(throwable, + cause -> DittoInternalErrorException.newBuilder() + .dittoHeaders(subscribe.getDittoHeaders()) + .cause(cause) + .build() + ); + return Source.failed(dittoRuntimeException); + } + }) + ); + } + + private String nextSubscriptionId(final SubscribeForPersistedEvents subscribeForPersistedEvents) { + final String prefix = subscribeForPersistedEvents.getPrefix().orElse(""); + return prefix + subscriptionIdCounter++; + } + + /** + * Make a source that never completes until downstream request. + * + * @param upstream the source to lazify. + * @param the type of elements. + * @return the lazified source. + */ + private static Source lazify(final Source upstream) { + return Source.lazySource(() -> upstream); + } + +} diff --git a/edge/service/src/test/java/org/eclipse/ditto/edge/service/placeholders/ImmutableFeaturePlaceholderTest.java b/edge/service/src/test/java/org/eclipse/ditto/edge/service/placeholders/ImmutableFeaturePlaceholderTest.java index a4d0a2c32cb..e13b8a91218 100644 --- a/edge/service/src/test/java/org/eclipse/ditto/edge/service/placeholders/ImmutableFeaturePlaceholderTest.java +++ b/edge/service/src/test/java/org/eclipse/ditto/edge/service/placeholders/ImmutableFeaturePlaceholderTest.java @@ -146,7 +146,7 @@ public void testReplaceFeatureIdsForMergeThingOnTopLevelWithNonObjectValue() { .set(ThingCommand.JsonFields.TYPE, MergeThing.TYPE) .set(ThingCommand.JsonFields.JSON_THING_ID, THING_ID.toString()) .set("path", "/") - .set("value", "This is wrong.") + .set("value", JsonObject.newBuilder().set("foo", "This is wrong.").build()) .build(), DittoHeaders.empty()); assertThat(UNDER_TEST.resolveValues(mergeThing, "id")).isEmpty(); } diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/actors/AbstractConnectionsRetrievalActor.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/actors/AbstractConnectionsRetrievalActor.java index 0805ae353ea..6c4dde22a86 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/actors/AbstractConnectionsRetrievalActor.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/actors/AbstractConnectionsRetrievalActor.java @@ -24,6 +24,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import javax.annotation.Nullable; + import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; @@ -46,6 +48,7 @@ import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.json.JsonArray; import org.eclipse.ditto.json.JsonCollectors; +import org.eclipse.ditto.json.JsonFieldSelector; import akka.actor.AbstractActor; import akka.actor.ActorRef; @@ -111,7 +114,10 @@ protected void retrieveConnectionsById(final RetrieveAllConnectionIdsResponse al retrieveConnections(connectionIds .stream() .map(ConnectionId::of) - .toList(), initialCommand.getDittoHeaders()) + .toList(), + initialCommand.getSelectedFields().orElse(null), + initialCommand.getDittoHeaders() + ) .thenAccept(retrieveConnectionsResponse -> { sender.tell(retrieveConnectionsResponse, getSelf()); stop(); @@ -147,13 +153,15 @@ private void handleRetrieveConnections(final RetrieveConnections retrieveConnect } private CompletionStage retrieveConnections( - final Collection connectionIds, final DittoHeaders dittoHeaders) { + final Collection connectionIds, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { checkNotNull(connectionIds, "connectionIds"); checkNotNull(dittoHeaders, "dittoHeaders"); final List> completableFutures = connectionIds.parallelStream() - .map(connectionId -> retrieveConnection(RetrieveConnection.of(connectionId, dittoHeaders))) + .map(connectionId -> retrieveConnection(RetrieveConnection.of(connectionId, selectedFields, dittoHeaders))) .map(CompletionStage::toCompletableFuture) .toList(); diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/directives/auth/DevopsAuthenticationDirectiveFactory.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/directives/auth/DevopsAuthenticationDirectiveFactory.java index 5a487b95537..19e0ec35f62 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/directives/auth/DevopsAuthenticationDirectiveFactory.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/directives/auth/DevopsAuthenticationDirectiveFactory.java @@ -40,7 +40,7 @@ public static DevopsAuthenticationDirectiveFactory newInstance( } public DevopsAuthenticationDirective status() { - if (!devOpsConfig.isSecured()) { + if (!devOpsConfig.isSecured() || !devOpsConfig.isStatusSecured()) { return DevOpsInsecureAuthenticationDirective.getInstance(); } switch (devOpsConfig.getStatusAuthenticationMethod()) { diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/AbstractRoute.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/AbstractRoute.java index 43c2f5bcf81..2ad53e14bfd 100755 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/AbstractRoute.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/AbstractRoute.java @@ -17,6 +17,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.time.Duration; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -136,11 +137,15 @@ private static Attributes createSupervisionStrategy() { /** * Calculates a JsonFieldSelector from the passed {@code fieldsString}. * - * @param fieldsString the fields as string. + * @param fields the fields as potentially comma separated strings. * @return the Optional JsonFieldSelector */ - protected static Optional calculateSelectedFields(final Optional fieldsString) { - return fieldsString.map(fs -> JsonFactory.newFieldSelector(fs, JSON_FIELD_SELECTOR_PARSE_OPTIONS)); + protected static Optional calculateSelectedFields(final List fields) { + if (fields.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(JsonFactory.newFieldSelector(fields, JSON_FIELD_SELECTOR_PARSE_OPTIONS)); + } } /** diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsParameter.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsParameter.java index 93da53e6d70..d07da3c7272 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsParameter.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsParameter.java @@ -22,7 +22,13 @@ public enum ConnectionsParameter { * Request parameter for doing a dry-run before creating a connection. */ DRY_RUN("dry-run"), - IDS_ONLY("ids-only"); + + IDS_ONLY("ids-only"), + + /** + * Request parameter for including only the selected fields in the Connection JSON document(s). + */ + FIELDS("fields"); private final String parameterValue; diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsRoute.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsRoute.java index 631d66c73a4..5b29f630f8e 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsRoute.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsRoute.java @@ -51,8 +51,10 @@ import org.eclipse.ditto.gateway.service.endpoints.routes.AbstractRoute; import org.eclipse.ditto.gateway.service.endpoints.routes.RouteBaseProperties; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; import akka.http.javadsl.model.MediaTypes; import akka.http.javadsl.server.PathMatchers; @@ -142,11 +144,25 @@ private Route connections(final RequestContext ctx, final DittoHeaders dittoHead return pathEndOrSingleSlash(() -> concat( get(() -> // GET /connections?ids-only=false - parameterOptional(ConnectionsParameter.IDS_ONLY.toString(), idsOnly -> handlePerRequest(ctx, - RetrieveConnections.newInstance(idsOnly.map(Boolean::valueOf).orElse(false), - dittoHeaders) - )) - + parameterOptional(ConnectionsParameter.IDS_ONLY.toString(), idsOnly -> + parameterList(ConnectionsParameter.FIELDS.toString(), fields -> + { + final Optional selectedFields = + calculateSelectedFields(fields); + return handlePerRequest(ctx, RetrieveConnections.newInstance( + idsOnly.map(Boolean::valueOf).orElseGet(() -> selectedFields + .filter(sf -> sf.getPointers().size() == 1 && + sf.getPointers().contains( + JsonPointer.of("id") + ) + ).isPresent() + ), + selectedFields.orElse(null), + dittoHeaders) + ); + } + ) + ) ), post(() -> // POST /connections?dry-run= parameterOptional(ConnectionsParameter.DRY_RUN.toString(), dryRun -> @@ -198,13 +214,17 @@ private Route connectionsEntry(final RequestContext ctx, payloadSource -> handlePerRequest(ctx, dittoHeaders, payloadSource, payloadJsonString -> ModifyConnection.of( - buildConnectionForPut(connectionId, - payloadJsonString), + buildConnectionForPut(connectionId, payloadJsonString), dittoHeaders)) ) ), get(() -> // GET /connections/ - handlePerRequest(ctx, RetrieveConnection.of(connectionId, dittoHeaders)) + parameterList(ConnectionsParameter.FIELDS.toString(), fields -> + handlePerRequest(ctx, RetrieveConnection.of(connectionId, + calculateSelectedFields(fields).orElse(null), + dittoHeaders) + ) + ) ), delete(() -> // DELETE /connections/ handlePerRequest(ctx, DeleteConnection.of(connectionId, dittoHeaders)) diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/policies/PoliciesRoute.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/policies/PoliciesRoute.java index 1a0a06065b7..8b8d345bc04 100755 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/policies/PoliciesRoute.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/policies/PoliciesRoute.java @@ -150,9 +150,9 @@ private Route policyId(final RequestContext ctx, final DittoHeaders dittoHeaders return pathEndOrSingleSlash(() -> concat( // GET /policies/?fields= - get(() -> parameterOptional(PoliciesParameter.FIELDS.toString(), fieldsString -> + get(() -> parameterList(PoliciesParameter.FIELDS.toString(), fields -> handlePerRequest(ctx, RetrievePolicy.of(policyId, dittoHeaders, - calculateSelectedFields(fieldsString).orElse(null))) + calculateSelectedFields(fields).orElse(null))) )), put(() -> // PUT /policies/ ensureMediaTypeJsonWithFallbacksThenExtractDataBytes(ctx, dittoHeaders, diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/sse/ThingsSseRouteBuilder.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/sse/ThingsSseRouteBuilder.java index 83141a79a5e..cea73da80ee 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/sse/ThingsSseRouteBuilder.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/sse/ThingsSseRouteBuilder.java @@ -18,6 +18,7 @@ import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.time.Duration; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -41,7 +42,10 @@ import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.SignalEnrichmentFailedException; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.FeatureToggle; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.UriEncoding; import org.eclipse.ditto.gateway.service.endpoints.routes.AbstractRoute; import org.eclipse.ditto.gateway.service.endpoints.routes.things.ThingsParameter; @@ -59,8 +63,13 @@ import org.eclipse.ditto.internal.utils.metrics.instruments.counter.Counter; import org.eclipse.ditto.internal.utils.pubsub.StreamingType; import org.eclipse.ditto.internal.utils.search.SearchSource; +import org.eclipse.ditto.json.JsonCollectors; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.json.JsonFieldSelector; +import org.eclipse.ditto.json.JsonKey; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.json.JsonValue; import org.eclipse.ditto.messages.model.Message; @@ -118,10 +127,19 @@ public final class ThingsSseRouteBuilder extends RouteDirectives implements SseR private static final String LAST_EVENT_ID_HEADER = "Last-Event-ID"; private static final String PARAM_FILTER = "filter"; - private static final String PARAM_FIELDS = "fields"; + private static final String PARAM_FIELDS = ThingsParameter.FIELDS.toString(); private static final String PARAM_OPTION = "option"; private static final String PARAM_NAMESPACES = "namespaces"; private static final String PARAM_EXTRA_FIELDS = "extraFields"; + + private static final String PARAM_FROM_HISTORICAL_REVISION = "from-historical-revision"; + private static final String PARAM_TO_HISTORICAL_REVISION = "to-historical-revision"; + private static final String PARAM_FROM_HISTORICAL_TIMESTAMP = "from-historical-timestamp"; + private static final String PARAM_TO_HISTORICAL_TIMESTAMP = "to-historical-timestamp"; + + private static final JsonFieldDefinition CONTEXT = + JsonFactory.newJsonObjectFieldDefinition("_context"); + private static final PartialFunction ACCEPT_HEADER_EXTRACTOR = newAcceptHeaderExtractor(); private static final Counter THINGS_SSE_COUNTER = getCounterFor(PATH_THINGS); @@ -249,7 +267,7 @@ private Route buildThingsSseRoute(final RequestContext ctx, // /things/ rawPathPrefix(PathMatchers.slash().concat(PathMatchers.segment()), thingId -> parameterMap(parameters -> { - final HashMap params = new HashMap<>(parameters); + final Map params = new HashMap<>(parameters); params.put(ThingsParameter.IDS.toString(), thingId); return concat( @@ -268,8 +286,7 @@ private Route buildThingsSseRoute(final RequestContext ctx, return createMessagesSseRoute(ctx, dhcs, thingId, jsonPointerString); } else { - params.put(ThingsParameter.FIELDS.toString(), - jsonPointerString); + params.put(PARAM_FIELDS, jsonPointerString); return createSseRoute(ctx, dhcs, JsonPointer.of(jsonPointerString), params @@ -318,13 +335,32 @@ private Route createSseRoute(final RequestContext ctx, final CompletionStage namespaces = getNamespaces(parameters.get(PARAM_NAMESPACES)); final List targetThingIds = getThingIds(parameters.get(ThingsParameter.IDS.toString())); - @Nullable final ThingFieldSelector fields = - getFieldSelector(parameters.get(ThingsParameter.FIELDS.toString())); + @Nullable final ThingFieldSelector fields = getFieldSelector(parameters.get(PARAM_FIELDS)); @Nullable final ThingFieldSelector extraFields = getFieldSelector(parameters.get(PARAM_EXTRA_FIELDS)); + + @Nullable final Long fromHistoricalRevision = Optional.ofNullable( + parameters.get(PARAM_FROM_HISTORICAL_REVISION)) + .map(Long::parseLong) + .orElse(null); + @Nullable final Long toHistoricalRevision = Optional.ofNullable( + parameters.get(PARAM_TO_HISTORICAL_REVISION)) + .map(Long::parseLong) + .orElse(null); + + @Nullable final Instant fromHistoricalTimestamp = Optional.ofNullable( + parameters.get(PARAM_FROM_HISTORICAL_TIMESTAMP)) + .map(Instant::parse) + .orElse(null); + @Nullable final Instant toHistoricalTimestamp = Optional.ofNullable( + parameters.get(PARAM_TO_HISTORICAL_TIMESTAMP)) + .map(Instant::parse) + .orElse(null); + final CompletionStage facadeStage = signalEnrichmentProvider == null ? CompletableFuture.completedStage(null) : signalEnrichmentProvider.getFacade(ctx.getRequest()); + final var sseSourceStage = facadeStage.thenCompose(facade -> dittoHeadersStage.thenCompose( dittoHeaders -> sseAuthorizationEnforcer.checkAuthorization(ctx, dittoHeaders).thenApply(unused -> { if (filterString != null) { @@ -332,6 +368,37 @@ private Route createSseRoute(final RequestContext ctx, final CompletionStage new IllegalStateException( + "Expected correlation-id in SSE DittoHeaders: " + dittoHeaders)); + final var authorizationContext = dittoHeaders.getAuthorizationContext(); + final Object startStreaming; + if (null != fromHistoricalRevision) { + FeatureToggle + .checkHistoricalApiAccessFeatureEnabled(SubscribeForPersistedEvents.TYPE, dittoHeaders); + startStreaming = SubscribeForPersistedEvents.of(targetThingIds.get(0), + fieldPointer, + fromHistoricalRevision, + null != toHistoricalRevision ? toHistoricalRevision : Long.MAX_VALUE, + dittoHeaders); + } else if (null != fromHistoricalTimestamp) { + FeatureToggle + .checkHistoricalApiAccessFeatureEnabled(SubscribeForPersistedEvents.TYPE, dittoHeaders); + startStreaming = SubscribeForPersistedEvents.of(targetThingIds.get(0), + fieldPointer, + fromHistoricalTimestamp, + toHistoricalTimestamp, + dittoHeaders); + } else { + startStreaming = + StartStreaming.getBuilder(StreamingType.EVENTS, connectionCorrelationId, + authorizationContext) + .withNamespaces(namespaces) + .withFilter(filterString) + .withExtraFields(extraFields) + .build(); + } + final Source publisherSource = SupervisedStream.sourceQueue(10); @@ -339,25 +406,14 @@ private Route createSseRoute(final RequestContext ctx, final CompletionStage { final SupervisedStream.WithQueue withQueue = pair.first(); final KillSwitch killSwitch = pair.second(); - final String connectionCorrelationId = dittoHeaders.getCorrelationId() - .orElseThrow(() -> new IllegalStateException( - "Expected correlation-id in SSE DittoHeaders: " + dittoHeaders)); final var jsonSchemaVersion = dittoHeaders.getSchemaVersion() .orElse(JsonSchemaVersion.LATEST); sseConnectionSupervisor.supervise(withQueue.getSupervisedStream(), connectionCorrelationId, dittoHeaders); - final var authorizationContext = dittoHeaders.getAuthorizationContext(); final var connect = new Connect(withQueue.getSourceQueue(), connectionCorrelationId, STREAMING_TYPE_SSE, jsonSchemaVersion, null, Set.of(), authorizationContext, null); - final var startStreaming = - StartStreaming.getBuilder(StreamingType.EVENTS, connectionCorrelationId, - authorizationContext) - .withNamespaces(namespaces) - .withFilter(filterString) - .withExtraFields(extraFields) - .build(); Patterns.ask(streamingActor, connect, LOCAL_ASK_TIMEOUT) .thenApply(ActorRef.class::cast) .thenAccept(streamingSessionActor -> @@ -369,8 +425,7 @@ private Route createSseRoute(final RequestContext ctx, final CompletionStage - postprocess(jsonifiable, facade, targetThingIds, namespaces, fieldPointer, - fields)) + postprocess(jsonifiable, facade, targetThingIds, namespaces, fieldPointer, fields)) .mapConcat(jsonValues -> jsonValues) .map(jsonValue -> { THINGS_SSE_COUNTER.increment(); @@ -546,8 +601,7 @@ private CompletionStage> postprocess(final SessionedJsonif final Supplier>> emptySupplier = () -> CompletableFuture.completedFuture(Collections.emptyList()); - if (jsonifiable.getJsonifiable() instanceof ThingEvent) { - final ThingEvent event = (ThingEvent) jsonifiable.getJsonifiable(); + if (jsonifiable.getJsonifiable() instanceof ThingEvent event) { final boolean isLiveEvent = StreamingType.isLiveSignal(event); if (!isLiveEvent && namespaceMatches(event, namespaces) && targetThingIdMatches(event, targetThingIds)) { @@ -640,11 +694,21 @@ private static Collection toNonemptyValue(final Thing thing, final Th final JsonObject thingJson = null != fields ? thing.toJson(jsonSchemaVersion, fields) : thing.toJson(jsonSchemaVersion); + @Nullable final JsonValue returnValue; if (!fieldPointer.isEmpty()) { returnValue = thingJson.getValue(fieldPointer).orElse(null); } else { - returnValue = thingJson; + final boolean includeContext = Optional.ofNullable(fields) + .filter(field -> field.getPointers().stream() + .map(JsonPointer::getRoot) + .anyMatch(p -> p.equals(CONTEXT.getPointer().getRoot())) + ).isPresent(); + if (includeContext) { + returnValue = addContext(thingJson.toBuilder(), event).get(fields); + } else { + returnValue = thingJson; + } } return (thingJson.isEmpty() || null == returnValue) ? Collections.emptyList() : Collections.singletonList(returnValue); @@ -694,4 +758,29 @@ private static Counter getCounterFor(final String path) { .tag("path", path); } + /** + * Add a JSON object at {@code _context} key containing e.g. the {@code headers} of the passed + * {@code withDittoHeaders}. + * + * @param objectBuilder the JsonObject build to add the {@code _context} to. + * @param withDittoHeaders the object to extract the {@code DittoHeaders} from. + * @return the built JsonObject including the {@code _context}. + */ + private static JsonObject addContext(final JsonObjectBuilder objectBuilder, + final WithDittoHeaders withDittoHeaders) { + + objectBuilder.set(CONTEXT, JsonObject.newBuilder() + .set("headers", dittoHeadersToJson(withDittoHeaders.getDittoHeaders())) + .build() + ); + return objectBuilder.build(); + } + + private static JsonObject dittoHeadersToJson(final DittoHeaders dittoHeaders) { + return dittoHeaders.entrySet() + .stream() + .map(entry -> JsonFactory.newField(JsonKey.of(entry.getKey()), JsonFactory.newValue(entry.getValue()))) + .collect(JsonCollectors.fieldsToObject()); + } + } diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/FeaturesRoute.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/FeaturesRoute.java index 200e7835f4e..62495f22262 100755 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/FeaturesRoute.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/FeaturesRoute.java @@ -105,9 +105,9 @@ private Route features(final RequestContext ctx, final DittoHeaders dittoHeaders return pathEndOrSingleSlash(() -> concat( // GET /features?fields= - get(() -> parameterOptional(ThingsParameter.FIELDS.toString(), fieldsString -> + get(() -> parameterList(ThingsParameter.FIELDS.toString(), fields -> handlePerRequest(ctx, RetrieveFeatures.of(thingId, - calculateSelectedFields(fieldsString).orElse(null), + calculateSelectedFields(fields).orElse(null), dittoHeaders)) ) ), @@ -143,9 +143,9 @@ private Route featuresEntry(final RequestContext ctx, final DittoHeaders dittoHe pathEndOrSingleSlash(() -> concat( // GET /features/{featureId}?fields= - get(() -> parameterOptional(ThingsParameter.FIELDS.toString(), - fieldsString -> handlePerRequest(ctx, RetrieveFeature.of(thingId, featureId, - calculateSelectedFields(fieldsString).orElse(null), + get(() -> parameterList(ThingsParameter.FIELDS.toString(), fields -> + handlePerRequest(ctx, RetrieveFeature.of(thingId, featureId, + calculateSelectedFields(fields).orElse(null), dittoHeaders)) ) ), @@ -235,10 +235,10 @@ private Route featuresEntryProperties(final RequestContext ctx, final DittoHeade pathEndOrSingleSlash(() -> concat( // GET /features/{featureId}/properties?fields= - get(() -> parameterOptional(ThingsParameter.FIELDS.toString(), - fieldsString -> handlePerRequest(ctx, + get(() -> parameterList(ThingsParameter.FIELDS.toString(), fields -> + handlePerRequest(ctx, RetrieveFeatureProperties.of(thingId, featureId, - calculateSelectedFields(fieldsString).orElse(null), + calculateSelectedFields(fields).orElse(null), dittoHeaders)) ) ), @@ -334,10 +334,10 @@ private Route featuresEntryDesiredProperties(final RequestContext ctx, final Dit pathEndOrSingleSlash(() -> concat( // GET /features/{featureId}/desiredProperties?fields= - get(() -> parameterOptional(ThingsParameter.FIELDS.toString(), - fieldsString -> handlePerRequest(ctx, + get(() -> parameterList(ThingsParameter.FIELDS.toString(), fields -> + handlePerRequest(ctx, RetrieveFeatureDesiredProperties.of(thingId, featureId, - calculateSelectedFields(fieldsString).orElse(null), + calculateSelectedFields(fields).orElse(null), dittoHeaders)) ) ), diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/ThingsRoute.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/ThingsRoute.java index 783c0fb5107..27f95b08d9c 100755 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/ThingsRoute.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/ThingsRoute.java @@ -14,12 +14,12 @@ import static org.eclipse.ditto.base.model.exceptions.DittoJsonException.wrapJsonRuntimeException; +import java.util.Arrays; import java.util.EnumMap; import java.util.List; import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; -import java.util.stream.Stream; import javax.annotation.Nullable; @@ -177,23 +177,29 @@ private Route things(final RequestContext ctx, final DittoHeaders dittoHeaders) } private Route buildRetrieveThingsRoute(final RequestContext ctx, final DittoHeaders dittoHeaders) { - // GET /things?ids=... - return parameter(ThingsParameter.IDS.toString(), idsString -> - parameterOptional(ThingsParameter.FIELDS.toString(), fieldsString -> - handlePerRequest(ctx, RetrieveThings.getBuilder(splitThingIdString(idsString)) - .selectedFields(calculateSelectedFields(fieldsString)) + + return parameterList(ThingsParameter.IDS.toString(), idsStrings -> { + if (!idsStrings.isEmpty()) { + // GET /things?ids=... + return parameterList(ThingsParameter.FIELDS.toString(), fields -> + handlePerRequest(ctx, RetrieveThings.getBuilder(splitThingIdStrings(idsStrings)) + .selectedFields(calculateSelectedFields(fields)) .dittoHeaders(dittoHeaders) .build(), (responseValue, response) -> - response.withEntity(determineResponseContentType(ctx), responseValue.toString()) + response.withEntity(determineResponseContentType(ctx), + responseValue.toString()) ) - ) - ).orElse( // GET /things - thingSearchParameterOptional(params -> - parameterOptional(ThingsParameter.FIELDS.toString(), fieldsString -> - handlePerRequest(ctx, QueryThings.of(null, // allow filter only on /search/things but not here - ThingSearchRoute.calculateOptions(params.get(ThingSearchParameter.OPTION)), - calculateSelectedFields(fieldsString).orElse(null), + ); + } else { + // GET /things + return thingSearchParameterOptional(params -> + parameterList(ThingsParameter.FIELDS.toString(), fields -> + handlePerRequest(ctx, + QueryThings.of(null, // allow filter only on /search/things but not here + ThingSearchRoute.calculateOptions( + params.get(ThingSearchParameter.OPTION)), + calculateSelectedFields(fields).orElse(null), ThingSearchRoute.calculateNamespaces( params.get(ThingSearchParameter.NAMESPACES)), dittoHeaders), @@ -201,25 +207,26 @@ private Route buildRetrieveThingsRoute(final RequestContext ctx, final DittoHead transformQueryThingsResult(ctx, responseValue, response) ) ) - ) - ); + ); + } + }); } private Route thingSearchParameterOptional( - final Function>, Route> inner) { + final Function>, Route> inner) { return thingSearchParameterOptionalImpl(ThingSearchParameter.values(), new EnumMap<>(ThingSearchParameter.class), inner); } private Route thingSearchParameterOptionalImpl(final ThingSearchParameter[] values, - final EnumMap> accumulator, - final Function>, Route> inner) { + final EnumMap> accumulator, + final Function>, Route> inner) { if (accumulator.size() >= values.length) { return inner.apply(accumulator); } else { final ThingSearchParameter parameter = values[accumulator.size()]; - return parameterOptional(parameter.toString(), parameterValueOptional -> { - accumulator.put(parameter, parameterValueOptional); + return parameterList(parameter.toString(), parameterValues -> { + accumulator.put(parameter, parameterValues); return thingSearchParameterOptionalImpl(values, accumulator, inner); }); } @@ -271,12 +278,13 @@ private static akka.http.javadsl.model.ContentType.NonBinary determineResponseCo return contentType; } - private static List splitThingIdString(final String thingIdString) { + private static List splitThingIdStrings(final List thingIdStrings) { final List result; - if (thingIdString.isEmpty()) { + if (thingIdStrings.isEmpty()) { result = List.of(); } else { - result = Stream.of(thingIdString.split(",")) + result = thingIdStrings.stream() + .flatMap(tid -> Arrays.stream(tid.split(","))) .map(ThingId::of) .toList(); } @@ -329,9 +337,9 @@ private Route thingsEntry(final RequestContext ctx, final DittoHeaders dittoHead return pathEndOrSingleSlash(() -> concat( // GET /things/?fields= - get(() -> parameterOptional(ThingsParameter.FIELDS.toString(), fieldsString -> + get(() -> parameterList(ThingsParameter.FIELDS.toString(), fields -> handlePerRequest(ctx, RetrieveThing.getBuilder(thingId, dittoHeaders) - .withSelectedFields(calculateSelectedFields(fieldsString) + .withSelectedFields(calculateSelectedFields(fields) .orElse(null)) .build()) ) @@ -352,6 +360,8 @@ private Route thingsEntry(final RequestContext ctx, final DittoHeaders dittoHead payloadSource -> handlePerRequest(ctx, dittoHeaders, payloadSource, thingJson -> MergeThing.withThing(thingId, thingFromJsonForPatch(thingJson, thingId, dittoHeaders), + createInlinePolicyJson(thingJson), + getCopyPolicyFrom(thingJson), dittoHeaders))) ), // DELETE /things/ @@ -431,9 +441,9 @@ private Route thingsEntryAttributes(final RequestContext ctx, final DittoHeaders pathEndOrSingleSlash(() -> concat( // GET /things//attributes?fields= - get(() -> parameterOptional(ThingsParameter.FIELDS.toString(), fieldsString -> + get(() -> parameterList(ThingsParameter.FIELDS.toString(), fields -> handlePerRequest(ctx, RetrieveAttributes.of(thingId, - calculateSelectedFields(fieldsString).orElse(null), + calculateSelectedFields(fields).orElse(null), dittoHeaders)) ) ), diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/thingsearch/ThingSearchRoute.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/thingsearch/ThingSearchRoute.java index 5998ad37abc..d69430355fa 100755 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/thingsearch/ThingSearchRoute.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/thingsearch/ThingSearchRoute.java @@ -15,10 +15,12 @@ import java.util.Arrays; import java.util.EnumMap; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.annotation.Nullable; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.gateway.service.endpoints.routes.AbstractRoute; @@ -104,47 +106,59 @@ private Route searchThings(final RequestContext ctx, final DittoHeaders dittoHea } private Route thingSearchParameterOptional( - final Function>, Route> inner) { + final Function>, Route> inner) { return thingSearchParameterOptionalImpl(ThingSearchParameter.values(), new EnumMap<>(ThingSearchParameter.class), inner); } private Route thingSearchParameterOptionalImpl(final ThingSearchParameter[] values, - final EnumMap> accumulator, - final Function>, Route> inner) { + final EnumMap> accumulator, + final Function>, Route> inner) { if (accumulator.size() >= values.length) { return inner.apply(accumulator); } else { final ThingSearchParameter parameter = values[accumulator.size()]; - return parameterOptional(parameter.toString(), parameterValueOptional -> { - accumulator.put(parameter, parameterValueOptional); + return parameterList(parameter.toString(), parameterValues -> { + accumulator.put(parameter, parameterValues); return thingSearchParameterOptionalImpl(values, accumulator, inner); }); } } - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - private static String calculateFilter(final Optional filterString) { - return filterString.orElse(null); + @Nullable + private static String calculateFilter(final List filterString) { + if (filterString.isEmpty()) { + return null; + } + return filterString.stream() + .collect(Collectors.joining(",", "and(", ")")); } - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - public static Set calculateNamespaces(final Optional namespacesString) { - final Function> splitAndRemoveEmpty = + @Nullable + public static Set calculateNamespaces(final List namespacesStrings) { + final Function> splitAndRemoveEmpty = s -> Arrays.stream(s.split(",")) - .filter(segment -> !segment.isEmpty()) - .collect(Collectors.toSet()); + .filter(segment -> !segment.isEmpty()); // if no namespaces are given explicitly via query parameter, // return null to signify the lack of namespace restriction - return namespacesString.map(splitAndRemoveEmpty).orElse(null); + if (namespacesStrings.isEmpty()) { + return null; + } + return namespacesStrings.stream() + .flatMap(splitAndRemoveEmpty) + .collect(Collectors.toSet()); } - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - public static List calculateOptions(final Optional optionsString) { + @Nullable + public static List calculateOptions(final List optionsString) { + if (optionsString.isEmpty()) { + return null; + } return optionsString - .map(s -> Arrays.asList(s.split(","))) - .orElse(null); + .stream() + .flatMap(s -> Arrays.stream(s.split(","))) + .toList(); } } diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SessionedJsonifiable.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SessionedJsonifiable.java index 111735f579d..a21ca920ab4 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SessionedJsonifiable.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SessionedJsonifiable.java @@ -22,6 +22,7 @@ import org.eclipse.ditto.base.model.json.Jsonifiable; import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.gateway.service.streaming.signals.StreamingAck; import org.eclipse.ditto.internal.models.signalenrichment.SignalEnrichmentFacade; import org.eclipse.ditto.internal.utils.pubsub.StreamingType; @@ -141,16 +142,27 @@ static SessionedJsonifiable response(final CommandResponse response) { } /** - * Create a sessioned Jsonifiable for a {@link org.eclipse.ditto.thingsearch.model.signals.events.SubscriptionEvent} + * Create a sessioned Jsonifiable for a {@link SubscriptionEvent} * as response. * - * @param subscriptionEvent the {@link org.eclipse.ditto.thingsearch.model.signals.events.SubscriptionEvent} as response. + * @param subscriptionEvent the {@link SubscriptionEvent} as response. * @return the sessioned Jsonifiable. */ static SessionedJsonifiable subscription(final SubscriptionEvent subscriptionEvent) { return new SessionedResponseErrorOrAck(subscriptionEvent, subscriptionEvent.getDittoHeaders(), null); } + /** + * Create a sessioned Jsonifiable for a {@link StreamingSubscriptionEvent} + * as response. + * + * @param streamingSubscriptionEvent the {@link StreamingSubscriptionEvent} as response. + * @return the sessioned Jsonifiable. + */ + static SessionedJsonifiable streamingSubscription(final StreamingSubscriptionEvent streamingSubscriptionEvent) { + return new SessionedResponseErrorOrAck(streamingSubscriptionEvent, streamingSubscriptionEvent.getDittoHeaders(), null); + } + /** * Create a sessioned Jsonifiable for a stream acknowledgement. * diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingActor.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingActor.java index 09a96f16998..b86130cdf43 100755 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingActor.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingActor.java @@ -17,6 +17,7 @@ import java.util.stream.StreamSupport; import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.edge.service.streaming.StreamingSubscriptionManager; import org.eclipse.ditto.gateway.service.security.authentication.jwt.JwtAuthenticationResultProvider; import org.eclipse.ditto.gateway.service.security.authentication.jwt.JwtValidator; import org.eclipse.ditto.gateway.service.streaming.signals.Connect; @@ -61,6 +62,7 @@ public final class StreamingActor extends AbstractActorWithTimers implements Ret private final JwtValidator jwtValidator; private final JwtAuthenticationResultProvider jwtAuthenticationResultProvider; private final Props subscriptionManagerProps; + private final Props streamingSubscriptionManagerProps; private final DittoDiagnosticLoggingAdapter logger = DittoLoggerFactory.getDiagnosticLoggingAdapter(this); private final HeaderTranslator headerTranslator; private int childCounter = -1; @@ -94,9 +96,13 @@ private StreamingActor(final DittoProtocolSub dittoProtocolSub, this.headerTranslator = headerTranslator; streamingSessionsCounter = DittoMetrics.gauge("streaming_sessions_count"); final ActorSelection commandForwarderSelection = ActorSelection.apply(commandForwarder, ""); + final Materializer materializer = Materializer.createMaterializer(getContext()); subscriptionManagerProps = SubscriptionManager.props(streamingConfig.getSearchIdleTimeout(), pubSubMediator, - commandForwarderSelection, Materializer.createMaterializer(getContext())); + commandForwarderSelection, materializer); + streamingSubscriptionManagerProps = + StreamingSubscriptionManager.props(streamingConfig.getSearchIdleTimeout(), + commandForwarderSelection, materializer); scheduleScrapeStreamSessionsCounter(); } @@ -148,7 +154,8 @@ private Receive createConnectAndMetricsBehavior() { final ActorRef streamingSessionActor = getContext().actorOf( StreamingSessionActor.props(connect, dittoProtocolSub, commandRouter, streamingConfig, headerTranslator, - subscriptionManagerProps, jwtValidator, jwtAuthenticationResultProvider), + subscriptionManagerProps, streamingSubscriptionManagerProps, + jwtValidator, jwtAuthenticationResultProvider), sessionActorName); getSender().tell(streamingSessionActor, ActorRef.noSender()); }) diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActor.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActor.java index 48f5b64d661..d847fe6c9ce 100755 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActor.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActor.java @@ -27,8 +27,10 @@ import org.eclipse.ditto.base.model.acks.AcknowledgementLabel; import org.eclipse.ditto.base.model.acks.AcknowledgementLabelNotDeclaredException; import org.eclipse.ditto.base.model.acks.AcknowledgementLabelNotUniqueException; +import org.eclipse.ditto.base.model.acks.DittoAcknowledgementLabel; import org.eclipse.ditto.base.model.acks.FatalPubSubException; import org.eclipse.ditto.base.model.auth.AuthorizationContext; +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.entity.id.WithEntityId; import org.eclipse.ditto.base.model.exceptions.DittoHeaderInvalidException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; @@ -41,7 +43,10 @@ import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.edge.service.acknowledgements.AcknowledgementAggregatorActorStarter; import org.eclipse.ditto.edge.service.acknowledgements.AcknowledgementForwarderActor; import org.eclipse.ditto.edge.service.acknowledgements.message.MessageCommandAckRequestSetter; @@ -50,6 +55,7 @@ import org.eclipse.ditto.edge.service.acknowledgements.things.ThingLiveCommandAckRequestSetter; import org.eclipse.ditto.edge.service.acknowledgements.things.ThingModifyCommandAckRequestSetter; import org.eclipse.ditto.edge.service.placeholders.EntityIdPlaceholder; +import org.eclipse.ditto.edge.service.streaming.StreamingSubscriptionManager; import org.eclipse.ditto.gateway.api.GatewayInternalErrorException; import org.eclipse.ditto.gateway.api.GatewayWebsocketSessionAbortedException; import org.eclipse.ditto.gateway.api.GatewayWebsocketSessionClosedException; @@ -92,6 +98,9 @@ import akka.japi.pf.ReceiveBuilder; import akka.pattern.Patterns; import akka.stream.KillSwitch; +import akka.stream.SourceRef; +import akka.stream.javadsl.Keep; +import akka.stream.javadsl.Sink; import akka.stream.javadsl.SourceQueueWithComplete; import scala.PartialFunction; @@ -120,6 +129,7 @@ final class StreamingSessionActor extends AbstractActorWithTimers { private final ActorRef commandForwarder; private final StreamingConfig streamingConfig; private final ActorRef subscriptionManager; + private final ActorRef streamingSubscriptionManager; private final Set outstandingSubscriptionAcks; private final Map streamingSessions; private final JwtValidator jwtValidator; @@ -139,6 +149,7 @@ private StreamingSessionActor(final Connect connect, final StreamingConfig streamingConfig, final HeaderTranslator headerTranslator, final Props subscriptionManagerProps, + final Props streamingSubscriptionManagerProps, final JwtValidator jwtValidator, final JwtAuthenticationResultProvider jwtAuthenticationResultProvider) { @@ -172,6 +183,8 @@ private StreamingSessionActor(final Connect connect, .withCorrelationId(connectionCorrelationId); connect.getSessionExpirationTime().ifPresent(this::startSessionTimeout); subscriptionManager = getContext().actorOf(subscriptionManagerProps, SubscriptionManager.ACTOR_NAME); + streamingSubscriptionManager = getContext().actorOf(streamingSubscriptionManagerProps, + StreamingSubscriptionManager.ACTOR_NAME); declaredAcks = connect.getDeclaredAcknowledgementLabels(); startSubscriptionRefreshTimer(); } @@ -185,6 +198,7 @@ private StreamingSessionActor(final Connect connect, * @param streamingConfig the config to apply for the streaming session. * @param headerTranslator translates headers from external sources or to external sources. * @param subscriptionManagerProps Props of the subscription manager for search protocol. + * @param streamingSubscriptionManagerProps Props of the subscription manager for streaming subscription commands. * @param jwtValidator validator of JWT tokens. * @param jwtAuthenticationResultProvider provider of JWT authentication results. * @return the Akka configuration Props object. @@ -195,6 +209,7 @@ static Props props(final Connect connect, final StreamingConfig streamingConfig, final HeaderTranslator headerTranslator, final Props subscriptionManagerProps, + final Props streamingSubscriptionManagerProps, final JwtValidator jwtValidator, final JwtAuthenticationResultProvider jwtAuthenticationResultProvider) { @@ -205,6 +220,7 @@ static Props props(final Connect connect, streamingConfig, headerTranslator, subscriptionManagerProps, + streamingSubscriptionManagerProps, jwtValidator, jwtAuthenticationResultProvider); } @@ -253,12 +269,14 @@ private Receive createIncomingSignalBehavior() { .build(); final Receive signalBehavior = ReceiveBuilder.create() + .match(Acknowledgement.class, this::isWeakAckForBuiltInAckLabel, this::dropWeakAckForBuiltInAckLabelAcknowledgement) .match(Acknowledgement.class, this::hasUndeclaredAckLabel, this::ackLabelNotDeclared) .match(Acknowledgement.class, this::forwardAcknowledgementOrLiveCommandResponse) .match(CommandResponse.class, CommandResponse::isLiveCommandResponse, liveCommandResponse -> commandForwarder.forward(liveCommandResponse, getContext())) .match(CommandResponse.class, this::forwardAcknowledgementOrLiveCommandResponse) .match(ThingSearchCommand.class, this::forwardSearchCommand) + .match(StreamingSubscriptionCommand.class, this::forwardStreamingSubscriptionCommand) .match(Signal.class, signal -> // forward signals for which no reply is expected with self return address for downstream errors commandForwarder.tell(signal, getReturnAddress(signal))) @@ -279,6 +297,10 @@ private Receive createOutgoingSignalBehavior() { logger.debug("Got SubscriptionEvent in <{}> session, publishing: {}", type, signal); eventAndResponsePublisher.offer(SessionedJsonifiable.subscription(signal)); }) + .match(StreamingSubscriptionEvent.class, signal -> { + logger.debug("Got StreamingSubscriptionEvent in <{}> session, publishing: {}", type, signal); + eventAndResponsePublisher.offer(SessionedJsonifiable.streamingSubscription(signal)); + }) .match(CommandResponse.class, this::publishResponseOrError) .match(DittoRuntimeException.class, this::publishResponseOrError) .match(Signal.class, this::isSameOrigin, signal -> @@ -312,6 +334,44 @@ private Receive createOutgoingSignalBehavior() { private Receive createPubSubBehavior() { return ReceiveBuilder.create() + .match(SubscribeForPersistedEvents.class, streamPersistedEvents -> { + authorizationContext = streamPersistedEvents.getDittoHeaders().getAuthorizationContext(); + final var session = StreamingSession.of( + streamPersistedEvents.getEntityId() instanceof NamespacedEntityId nsEid ? + List.of(nsEid.getNamespace()) : List.of(), + null, + null, + getSelf(), + logger); + streamingSessions.put(StreamingType.EVENTS, session); + + Patterns.ask(commandForwarder, streamPersistedEvents, streamPersistedEvents.getDittoHeaders() + .getTimeout() + .orElse(Duration.ofSeconds(5)) + ) + .thenApply(response -> (SourceRef) response) + .whenComplete((sourceRef, throwable) -> { + if (null != sourceRef) { + sourceRef.getSource() + .toMat(Sink.actorRef(getSelf(), Control.TERMINATED), Keep.left()) + .run(getContext().getSystem()); + } else if (null != throwable) { + final var dittoRuntimeException = DittoRuntimeException + .asDittoRuntimeException(throwable, + cause -> GatewayInternalErrorException.newBuilder() + .dittoHeaders(DittoHeaders.newBuilder() + .correlationId(connectionCorrelationId) + .build()) + .cause(cause) + .build() + ); + eventAndResponsePublisher.offer(SessionedJsonifiable.error(dittoRuntimeException)); + terminateWebsocketStream(); + } else { + terminateWebsocketStream(); + } + }); + }) .match(StartStreaming.class, startStreaming -> { authorizationContext = startStreaming.getAuthorizationContext(); Criteria criteria; @@ -435,6 +495,16 @@ private static Receive addPreprocessors(final List", ack.getLabel()); + } + private boolean hasUndeclaredAckLabel(final Acknowledgement acknowledgement) { return !declaredAcks.contains(acknowledgement.getLabel()); } @@ -555,6 +625,11 @@ private void forwardSearchCommand(final ThingSearchCommand searchCommand) { subscriptionManager.tell(searchCommand, getSelf()); } + private void forwardStreamingSubscriptionCommand( + final StreamingSubscriptionCommand streamingSubscriptionCommand) { + streamingSubscriptionManager.tell(streamingSubscriptionCommand, getSelf()); + } + private boolean isSessionAllowedToReceiveSignal(final Signal signal, final StreamingSession session, final StreamingType streamingType) { diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SupervisedStream.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SupervisedStream.java index 84d7ad0a9b2..134b7f6e2e0 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SupervisedStream.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SupervisedStream.java @@ -35,7 +35,7 @@ public interface SupervisedStream { * @return the source queue. */ static Source sourceQueue(final int queueSize) { - return Source.queue(queueSize, OverflowStrategy.fail().withLogLevel(Logging.WarningLevel())) + return Source.queue(queueSize, OverflowStrategy.backpressure().withLogLevel(Logging.WarningLevel())) .viaMat(KillSwitches.single(), Keep.both()) .mapMaterializedValue(pair -> { final SourceQueueWithComplete sourceQueue = pair.first(); diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfig.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfig.java index e974d5a7a93..5e94e545a09 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfig.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfig.java @@ -33,23 +33,25 @@ public final class DefaultDevOpsConfig implements DevOpsConfig { private static final String CONFIG_PATH = "devops"; - private final boolean secureStatus; + private final boolean secured; private final DevopsAuthenticationMethod devopsAuthenticationMethod; private final String password; private final Collection devopsOAuth2Subjects; + private final boolean statusSecured; private final DevopsAuthenticationMethod statusAuthenticationMethod; private final String statusPassword; private final Collection statusOAuth2Subjects; private final OAuthConfig oAuthConfig; private DefaultDevOpsConfig(final ConfigWithFallback configWithFallback) { - secureStatus = configWithFallback.getBoolean(DevOpsConfigValue.SECURED.getConfigPath()); + secured = configWithFallback.getBoolean(DevOpsConfigValue.SECURED.getConfigPath()); devopsAuthenticationMethod = getDevopsAuthenticationMethod(configWithFallback, DevOpsConfigValue.DEVOPS_AUTHENTICATION_METHOD); password = configWithFallback.getString(DevOpsConfigValue.PASSWORD.getConfigPath()); devopsOAuth2Subjects = Collections.unmodifiableList(new ArrayList<>( configWithFallback.getStringList(DevOpsConfigValue.DEVOPS_OAUTH2_SUBJECTS.getConfigPath()))); + statusSecured = configWithFallback.getBoolean(DevOpsConfigValue.STATUS_SECURED.getConfigPath()); statusAuthenticationMethod = getDevopsAuthenticationMethod(configWithFallback, DevOpsConfigValue.STATUS_AUTHENTICATION_METHOD); statusPassword = configWithFallback.getString(DevOpsConfigValue.STATUS_PASSWORD.getConfigPath()); @@ -84,7 +86,7 @@ public static DefaultDevOpsConfig of(final Config config) { @Override public boolean isSecured() { - return secureStatus; + return secured; } @Override @@ -102,6 +104,11 @@ public Collection getDevopsOAuth2Subjects() { return devopsOAuth2Subjects; } + @Override + public boolean isStatusSecured() { + return statusSecured; + } + @Override public DevopsAuthenticationMethod getStatusAuthenticationMethod() { return statusAuthenticationMethod; @@ -131,10 +138,11 @@ public boolean equals(@Nullable final Object o) { return false; } final DefaultDevOpsConfig that = (DefaultDevOpsConfig) o; - return Objects.equals(secureStatus, that.secureStatus) && + return Objects.equals(secured, that.secured) && Objects.equals(devopsAuthenticationMethod, that.devopsAuthenticationMethod) && Objects.equals(password, that.password) && Objects.equals(devopsOAuth2Subjects, that.devopsOAuth2Subjects) && + Objects.equals(statusSecured, that.statusSecured) && Objects.equals(statusAuthenticationMethod, that.statusAuthenticationMethod) && Objects.equals(statusOAuth2Subjects, that.statusOAuth2Subjects) && Objects.equals(statusPassword, that.statusPassword) && @@ -143,17 +151,18 @@ public boolean equals(@Nullable final Object o) { @Override public int hashCode() { - return Objects.hash(secureStatus, devopsAuthenticationMethod, password, devopsOAuth2Subjects, - statusAuthenticationMethod, statusPassword, statusOAuth2Subjects, oAuthConfig); + return Objects.hash(secured, devopsAuthenticationMethod, password, devopsOAuth2Subjects, + statusSecured, statusAuthenticationMethod, statusPassword, statusOAuth2Subjects, oAuthConfig); } @Override public String toString() { return getClass().getSimpleName() + " [" + - "secureStatus=" + secureStatus + + "secured=" + secured + ", devopsAuthenticationMethod=" + devopsAuthenticationMethod + ", password=*****" + ", devopsOAuth2Subject=" + devopsOAuth2Subjects + + ", statusSecured=" + statusSecured + ", statusAuthenticationMethod=" + statusAuthenticationMethod + ", statusPassword=*****" + ", statusOAuth2Subject=" + statusOAuth2Subjects + diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DevOpsConfig.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DevOpsConfig.java index 0c23be7a711..59931d28192 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DevOpsConfig.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DevOpsConfig.java @@ -53,6 +53,13 @@ public interface DevOpsConfig { */ Collection getDevopsOAuth2Subjects(); + /** + * Indicates whether the DevOps resource {@code /status} should be secured or not. + * + * @return {@code true} if {@code /status} should be secured, {@code false} else; + */ + boolean isStatusSecured(); + /** * Returns the authentication method for status resources. * @@ -107,6 +114,11 @@ enum DevOpsConfigValue implements KnownConfigValue { */ DEVOPS_OAUTH2_SUBJECTS("devops-oauth2-subjects", List.of()), + /** + * Determines whether DevOps resource {@code /status} should be secured or not. + */ + STATUS_SECURED("status-secured", true), + /** * The authentication method for status resources. */ @@ -125,7 +137,7 @@ enum DevOpsConfigValue implements KnownConfigValue { private final String path; private final Object defaultValue; - private DevOpsConfigValue(final String thePath, final Object theDefaultValue) { + DevOpsConfigValue(final String thePath, final Object theDefaultValue) { path = thePath; defaultValue = theDefaultValue; } diff --git a/gateway/service/src/main/resources/gateway.conf b/gateway/service/src/main/resources/gateway.conf index 00cb2f3971d..846bf229b0c 100755 --- a/gateway/service/src/main/resources/gateway.conf +++ b/gateway/service/src/main/resources/gateway.conf @@ -112,7 +112,9 @@ ditto { "live-channel-timeout-strategy", "allow-policy-lockout", "condition", - "live-channel-condition" + "live-channel-condition", + "at-historical-revision", + "at-historical-timestamp" ] } @@ -250,6 +252,12 @@ ditto { # "..auth0.com/" # "..auth0.com/" # ] + # auth-subjects = [ + # "{{ jwt:sub }}", + # "{{ jwt:sub }}/{{ jwt:scp }}", + # "{{ jwt:sub }}/{{ jwt:scp }}@{{ jwt:client_id }}", + # "{{ jwt:roles/support }}" + # ] #} google = { issuer = "accounts.google.com" @@ -273,8 +281,6 @@ ditto { secured = true # Backwardcompatibility fallback secured = ${?ditto.gateway.authentication.devops.securestatus} - # Backwardcompatibility fallback - secured = ${?DEVOPS_SECURE_STATUS} secured = ${?DEVOPS_SECURED} # default authentiation method for the devops route @@ -292,10 +298,35 @@ ditto { password = ${?DEVOPS_PASSWORD} # oauth2 auth + oauth { + # force protocol to HTTPS for security + protocol = "https" + protocol = ${?DEVOPS_OAUTH2_PROTOCOL} + + # configure the amount of clock skew in seconds to tolerate when verifying the local time against the exp + # and nbf claims + allowed-clock-skew = 10s + allowed-clock-skew = ${?DEVOPS_OAUTH2_ALLOWED_CLOCK_SKEW} + + # map of all supported OpenID Connect authorization servers + # issuer should not contain the protocol (e.g. https://) + openid-connect-issuers = { + # auth0 = { + # issuer = "..auth0.com/" + # issuers = [ + # "..auth0.com/" + # "..auth0.com/" + # ] + #} + } + } devops-oauth2-subjects = ${?DEVOPS_OAUTH2_SUBJECTS} + status-secured = true + status-secured = ${?DEVOPS_STATUS_SECURED} + # default authentiation method for the status route - # can be set to "basic" or "oauth" + # can be set to "basic" or "oauth2" status-authentication-method = "basic" # override by environment variable status-authentication-method = ${?STATUS_AUTHENTICATION_METHOD} diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/ThingsRouteTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/ThingsRouteTest.java index cc153945d4c..fddd9483eec 100755 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/ThingsRouteTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/endpoints/routes/things/ThingsRouteTest.java @@ -19,8 +19,8 @@ import org.eclipse.ditto.gateway.service.endpoints.EndpointTestConstants; import org.eclipse.ditto.json.JsonKey; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.things.model.ThingIdInvalidException; import org.eclipse.ditto.things.model.ThingsModelFactory; -import org.eclipse.ditto.things.model.signals.commands.exceptions.MissingThingIdsException; import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingNotCreatableException; import org.eclipse.ditto.things.model.signals.commands.modify.MergeThing; import org.eclipse.ditto.things.model.signals.commands.modify.ModifyPolicyId; @@ -158,7 +158,7 @@ public void putAndRetrieveNullDefinition() { public void getThingsWithEmptyIdsList() { final var result = underTest.run(HttpRequest.GET("/things?ids=")); result.assertStatusCode(StatusCodes.BAD_REQUEST); - final var expectedEx = MissingThingIdsException.newBuilder() + final var expectedEx = ThingIdInvalidException.newBuilder("") .dittoHeaders(dittoHeaders) .build(); result.assertEntity(expectedEx.toJsonString()); diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalCommandRegistryTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalCommandRegistryTest.java index 4d192f59369..df3a1b3353b 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalCommandRegistryTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalCommandRegistryTest.java @@ -17,6 +17,7 @@ import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoAddConnectionLogEntry; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionIdsByTag; @@ -67,7 +68,8 @@ public GatewayServiceGlobalCommandRegistryTest() { PublishSignal.class, ModifySplitBrainResolver.class, CleanupPersistence.class, - SudoAddConnectionLogEntry.class + SudoAddConnectionLogEntry.class, + SubscribeForPersistedEvents.class ); } diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalEventRegistryTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalEventRegistryTest.java index e7b03a4ae76..d658562e4aa 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalEventRegistryTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalEventRegistryTest.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.gateway.service.starter; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases; import org.eclipse.ditto.policies.model.signals.events.ResourceDeleted; @@ -29,7 +30,8 @@ public GatewayServiceGlobalEventRegistryTest() { FeatureDeleted.class, ThingsOutOfSync.class, SubscriptionCreated.class, - ThingSnapshotTaken.class + ThingSnapshotTaken.class, + StreamingSubscriptionComplete.class ); } diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorHeaderInteractionTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorHeaderInteractionTest.java index d891436f2c8..b3f11decf00 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorHeaderInteractionTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorHeaderInteractionTest.java @@ -116,6 +116,7 @@ public static Collection getParameters() { private final TestProbe eventResponsePublisherProbe = TestProbe.apply("eventAndResponsePublisher", actorSystem); private final TestProbe commandRouterProbe = TestProbe.apply("commandRouter", actorSystem); private final TestProbe subscriptionManagerProbe = TestProbe.apply("subscriptionManager", actorSystem); + private final TestProbe streamingSubscriptionManagerProbe = TestProbe.apply("streamingSubscriptionManager", actorSystem); private final DittoProtocolSub dittoProtocolSub = Mockito.mock(DittoProtocolSub.class); private final SourceQueueWithComplete sourceQueue; @@ -198,7 +199,9 @@ private ActorRef createStreamingSessionActor() { null); final Props props = StreamingSessionActor.props(connect, dittoProtocolSub, commandRouterProbe.ref(), DefaultStreamingConfig.of(ConfigFactory.empty()), HeaderTranslator.empty(), - Props.create(TestProbeForwarder.class, subscriptionManagerProbe), Mockito.mock(JwtValidator.class), + Props.create(TestProbeForwarder.class, subscriptionManagerProbe), + Props.create(TestProbeForwarder.class, streamingSubscriptionManagerProbe), + Mockito.mock(JwtValidator.class), Mockito.mock(JwtAuthenticationResultProvider.class)); final ActorRef createdActor = actorSystem.actorOf(props); createdActors.add(createdActor); diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorTest.java index 4ae7337b51f..fd498c2dc4a 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorTest.java @@ -397,6 +397,7 @@ private Props getProps(final String... declaredAcks) { DefaultStreamingConfig.of(ConfigFactory.empty()), HeaderTranslator.empty(), Props.create(Actor.class, () -> new TestActor(new LinkedBlockingDeque<>())), + Props.create(Actor.class, () -> new TestActor(new LinkedBlockingDeque<>())), mockValidator, mockAuthenticationResultProvider); } diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfigTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfigTest.java index 625a1be8054..ab867d08fd5 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfigTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfigTest.java @@ -65,6 +65,7 @@ public void underTestReturnsDefaultValuesIfBaseConfigWasEmpty() { DevOpsConfig.DevOpsConfigValue.DEVOPS_AUTHENTICATION_METHOD); assertDefaultValueFor(underTest.getPassword(), DevOpsConfig.DevOpsConfigValue.PASSWORD); assertDefaultValueFor(underTest.getDevopsOAuth2Subjects(), DevOpsConfig.DevOpsConfigValue.DEVOPS_OAUTH2_SUBJECTS); + assertDefaultValueFor(underTest.isStatusSecured(), DevOpsConfig.DevOpsConfigValue.STATUS_SECURED); assertDefaultValueFor(underTest.getStatusAuthenticationMethod().getMethodName(), DevOpsConfig.DevOpsConfigValue.STATUS_AUTHENTICATION_METHOD); assertDefaultValueFor(underTest.getStatusPassword(), DevOpsConfig.DevOpsConfigValue.STATUS_PASSWORD); @@ -87,6 +88,7 @@ public void underTestReturnsValuesOfConfigFile() { assertConfiguredValueFor(underTest.getPassword(), DevOpsConfig.DevOpsConfigValue.PASSWORD, "bumlux"); assertConfiguredValueFor(underTest.getDevopsOAuth2Subjects(), DevOpsConfig.DevOpsConfigValue.DEVOPS_OAUTH2_SUBJECTS, List.of("someissuer:a", "someissuer:b")); + assertConfiguredValueFor(underTest.isStatusSecured(), DevOpsConfig.DevOpsConfigValue.SECURED, false); assertConfiguredValueFor(underTest.getStatusAuthenticationMethod().getMethodName(), DevOpsConfig.DevOpsConfigValue.STATUS_AUTHENTICATION_METHOD, "oauth2"); assertConfiguredValueFor(underTest.getStatusPassword(), DevOpsConfig.DevOpsConfigValue.STATUS_PASSWORD, diff --git a/gateway/service/src/test/resources/devops-test.conf b/gateway/service/src/test/resources/devops-test.conf index 99edaf1b31e..d99ae282b42 100644 --- a/gateway/service/src/test/resources/devops-test.conf +++ b/gateway/service/src/test/resources/devops-test.conf @@ -3,6 +3,7 @@ devops { devops-authentication-method = "basic" password = "bumlux" devops-oauth2-subjects = ["someissuer:a","someissuer:b"] + status-secured = false status-authentication-method = "oauth2" statusPassword = "1234" status-oauth2-subjects = ["someissuer:c"] diff --git a/internal/utils/cache-loaders/src/main/java/org/eclipse/ditto/internal/utils/cacheloaders/AskWithRetry.java b/internal/utils/cache-loaders/src/main/java/org/eclipse/ditto/internal/utils/cacheloaders/AskWithRetry.java index e495c5a25b6..9fd19c46b45 100644 --- a/internal/utils/cache-loaders/src/main/java/org/eclipse/ditto/internal/utils/cacheloaders/AskWithRetry.java +++ b/internal/utils/cache-loaders/src/main/java/org/eclipse/ditto/internal/utils/cacheloaders/AskWithRetry.java @@ -25,6 +25,8 @@ import javax.annotation.Nullable; import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.WithEntityId; import org.eclipse.ditto.base.model.exceptions.AskException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; @@ -196,9 +198,21 @@ private static CompletionStage> createAskHandle(final ActorR ThreadSafeDittoLogger l = LOGGER; if (null != dittoHeaders) { l = LOGGER.withCorrelationId(dittoHeaders); + } else if (message instanceof WithDittoHeaders withDittoHeaders) { + l = LOGGER.withCorrelationId(withDittoHeaders.getDittoHeaders()); } - l.warn("Got AskTimeout during ask for message <{}> - retrying.. : <{}>", - message.getClass().getSimpleName(), throwable.getMessage()); + final EntityId entityId; + if (message instanceof WithEntityId withEntityId) { + entityId = withEntityId.getEntityId(); + } else { + entityId = null; + } + l.warn("Got AskTimeout during ask for message <{}> and entityId <{} / {}> - retrying.. : <{}>", + message.getClass().getSimpleName(), + entityId, + entityId != null ? entityId.getEntityType() : null, + throwable.getMessage() + ); } // all non-known RuntimeException should be handled by the "Patterns.retry" with a retry: throw new UnknownAskRuntimeException(throwable); diff --git a/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/ConditionalHeadersValidator.java b/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/ConditionalHeadersValidator.java index 04eb3f0c04f..2cc0ca4a451 100644 --- a/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/ConditionalHeadersValidator.java +++ b/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/ConditionalHeadersValidator.java @@ -19,6 +19,7 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; +import org.eclipse.ditto.base.model.entity.Entity; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; import org.eclipse.ditto.base.model.headers.DittoHeaders; @@ -47,8 +48,8 @@ public interface ValidationSettings { * @param actual the actual ETag value. * @return the builder. */ - DittoRuntimeExceptionBuilder createPreconditionFailedExceptionBuilder(final String conditionalHeaderName, - final String expected, final String actual); + DittoRuntimeExceptionBuilder createPreconditionFailedExceptionBuilder(String conditionalHeaderName, + String expected, String actual); /** * Returns a builder for a {@link DittoRuntimeException} in case status {@code 304 (Not Modified)} should be @@ -58,8 +59,18 @@ DittoRuntimeExceptionBuilder createPreconditionFailedExceptionBuilder(final S * @param matched the matched value. * @return the builder. */ - DittoRuntimeExceptionBuilder createPreconditionNotModifiedExceptionBuilder(final String expectedNotToMatch, - final String matched); + DittoRuntimeExceptionBuilder createPreconditionNotModifiedExceptionBuilder(String expectedNotToMatch, + String matched); + + /** + * Returns a builder for a {@link DittoRuntimeException} in case status {@code 304 (Not Modified)} should be + * returned for when an updated value was equal to its previous value and the {@code if-equal} condition was + * set to "skip". + * + * @return the builder. + * @since 3.3.0 + */ + DittoRuntimeExceptionBuilder createPreconditionNotModifiedForEqualityExceptionBuilder(); } private final ValidationSettings validationSettings; @@ -118,6 +129,26 @@ public void checkConditionalHeaders(final Command command, checkIfNoneMatch(command, currentETagValue); } + /** + * Checks if the in the given {@code command} contained + * {@link org.eclipse.ditto.base.model.headers.DittoHeaderDefinition#IF_EQUAL if-equal header} defines whether to + * skip an update for when the value to update is the same as before. + * Throws a "*PreconditionNotModifiedException" for the respective entity. + * + * @param command The command that potentially contains the if-equal header. + * @param currentEntity the current entity targeted by the given {@code command}. + * @throws org.eclipse.ditto.base.model.exceptions.DittoRuntimeException when a condition fails (the concrete + * subclass is defined by {@link ValidationSettings}). + */ + public > C applyIfEqualHeader(final C command, @Nullable final Entity currentEntity) { + + if (skipPreconditionHeaderCheck(command, null)) { + return command; + } + + return applyIfEqual(command, currentEntity); + } + private boolean skipPreconditionHeaderCheck(final Command command, @Nullable final EntityTag currentETagValue) { return (currentETagValue == null && @@ -150,6 +181,12 @@ private void checkIfNoneMatch(final Command command, @Nullable final EntityTa }); } + private > C applyIfEqual(final C command, @Nullable final Entity entity) { + return IfEqualPreconditionHeader.fromDittoHeaders(command, validationSettings) + .map(ifEqual -> ifEqual.handleCommand(() -> ifEqual.meetsConditionFor(entity))) + .orElse(command); + } + private DittoRuntimeException buildPreconditionFailedException( final PreconditionHeader preconditionHeader, final DittoHeaders dittoHeaders, @Nullable final EntityTag currentETagValue) { diff --git a/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/IfEqualPreconditionHeader.java b/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/IfEqualPreconditionHeader.java new file mode 100644 index 00000000000..98d673c57a5 --- /dev/null +++ b/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/IfEqualPreconditionHeader.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.internal.utils.headers.conditional; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.util.Comparator; +import java.util.Optional; +import java.util.function.BooleanSupplier; +import java.util.function.Predicate; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.Entity; +import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; +import org.eclipse.ditto.base.model.headers.IfEqual; +import org.eclipse.ditto.base.model.json.FieldType; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.WithOptionalEntity; +import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.json.JsonCollectors; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonKey; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonValue; + +/** + * Custom Ditto {@code if-equal} precondition header supporting available strategies defined in {@link org.eclipse.ditto.base.model.headers.IfEqual}. + * The default is to always {@link IfEqual#UPDATE update} the entity, no matter if the update will lead to the + * same state. Another option is to {@link IfEqual#SKIP skip} updating an entity when it would be {@code equal} + * after the change. + * + * @param the type of the handled {@link Command}. + * @since 3.3.0 + */ +@Immutable +public final class IfEqualPreconditionHeader> implements PreconditionHeader> { + + private static final String IF_EQUAL_KEY = DittoHeaderDefinition.IF_EQUAL.getKey(); + + private final C command; + private final IfEqual ifEqual; + private final ConditionalHeadersValidator.ValidationSettings validationSettings; + + private IfEqualPreconditionHeader(final C command, final IfEqual ifEqual, + final ConditionalHeadersValidator.ValidationSettings validationSettings) { + this.command = checkNotNull(command, "command"); + this.ifEqual = checkNotNull(ifEqual, "ifEqual"); + this.validationSettings = checkNotNull(validationSettings, "validationSettings"); + } + + /** + * Extracts an {@link IfEqualPreconditionHeader} from the given {@code command} if present. + * + * @param command The command containing potentially the {@code if-equal} and the value to modify + * @param validationSettings the settings. + * @param the type of the handled {@link Command}. + * @return Optional of {@link IfEqualPreconditionHeader}. Empty if the given {@code command} don't contain an + * {@code if-equal} or if it is not a modifying command. + */ + public static > Optional> fromDittoHeaders( + final C command, + final ConditionalHeadersValidator.ValidationSettings validationSettings) { + + final Command.Category category = command.getCategory(); + if (category == Command.Category.MODIFY || category == Command.Category.MERGE) { + return command.getDittoHeaders().getIfEqual() + .map(ifEqual -> new IfEqualPreconditionHeader<>(command, ifEqual, validationSettings)); + } else { + return Optional.empty(); + } + } + + @Override + public String getKey() { + return IF_EQUAL_KEY; + } + + @Override + public String getValue() { + return ifEqual.toString(); + } + + /** + * Indicates whether this {@link IfEqualPreconditionHeader} passes for the given {@code entity}. + * This means that the {@code command} field which is currently processed would not change the passed {@code entity}, + * so the targeted desired state change indicated by the command is already present. + * + * @param entity The entity for which the equality condition should be met. + * @return True if the equality condition is met. False if not. + */ + @Override + public boolean meetsConditionFor(@Nullable final Entity entity) { + + if (entity == null) { + return false; + } + + if (ifEqual == IfEqual.SKIP) { + if (command.getCategory() == Command.Category.MODIFY && + command instanceof WithOptionalEntity withOptionalEntity) { + return withOptionalEntity.getEntity() + .map(newValue -> { + final Predicate nonHiddenAndNamespace = + FieldType.notHidden() + .or(jsonField -> jsonField.getKey().equals(JsonKey.of("_namespace"))); + final Optional previousValue = entity.toJson(JsonSchemaVersion.LATEST, nonHiddenAndNamespace) + .getValue(command.getResourcePath()); + return previousValue.filter(jsonValue -> jsonValue.equals(newValue)) + .isPresent(); + }) + .orElse(false); + } else if (command.getCategory() == Command.Category.MERGE && + command instanceof WithOptionalEntity withOptionalEntity) { + return withOptionalEntity.getEntity() + .map(newValue -> { + final Optional previousValue = entity.toJson() + .getValue(command.getResourcePath()); + if (newValue.isObject()) { + return previousValue.filter(JsonValue::isObject) + .map(JsonValue::asObject) + .filter(previousObject -> { + final JsonObject patchedAndSortedNewObject = + JsonFactory.mergeJsonValues(newValue.asObject(), previousObject) + .asObject().stream() + .sorted(Comparator.comparing(j -> j.getKey().toString())) + .collect(JsonCollectors.fieldsToObject()); + final JsonObject sortedOldObject = previousObject.stream() + .sorted(Comparator.comparing(j -> j.getKey().toString())) + .collect(JsonCollectors.fieldsToObject()); + return patchedAndSortedNewObject.equals(sortedOldObject); + }).isPresent(); + } else { + return previousValue.filter(jsonValue -> jsonValue.equals(newValue)) + .isPresent(); + } + }) + .orElse(false); + } else { + // other commands to "MODIFY" and "MERGE" do never match the "if-equal" precondition header + return false; + } + } else { + // for previous default behavior, "if-equal: update", don't match: + return false; + } + } + + /** + * Handles the {@link #command} field of this class by invoking the passed {@code isCompletelyEqualSupplier} to + * check whether the affected entity would be completely equal after applying the {@link #command}. + * + * @return the potentially adjusted Command. + */ + C handleCommand(final BooleanSupplier isCompletelyEqualSupplier) { + + final C potentiallyAdjustedCommand; + if (ifEqual == IfEqual.UPDATE) { + // default behavior - no change, just use the complete modify command, not matter what: + potentiallyAdjustedCommand = command; + } else if (ifEqual == IfEqual.SKIP) { + // lazily check for equality as this might be expensive to do: + final boolean completelyEqual = isCompletelyEqualSupplier.getAsBoolean(); + final Command.Category category = command.getCategory(); + if (completelyEqual && + (category == Command.Category.MODIFY || category == Command.Category.MERGE)) { + potentiallyAdjustedCommand = respondWithNotModified(); + } else { + potentiallyAdjustedCommand = command; + } + } else { + potentiallyAdjustedCommand = command; + } + return potentiallyAdjustedCommand; + } + + private C respondWithNotModified() { + throw validationSettings + .createPreconditionNotModifiedForEqualityExceptionBuilder() + .dittoHeaders(command.getDittoHeaders()) + .build(); + } + +} diff --git a/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/PreconditionHeader.java b/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/PreconditionHeader.java index e016033ea55..8f6e32b2283 100644 --- a/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/PreconditionHeader.java +++ b/internal/utils/conditional-headers/src/main/java/org/eclipse/ditto/internal/utils/headers/conditional/PreconditionHeader.java @@ -41,5 +41,5 @@ public interface PreconditionHeader { * @param objectToCheck The object for which this {@link PreconditionHeader} should meet the condition. * @return True if this {@link PreconditionHeader} meets the condition. False if not. */ - boolean meetsConditionFor(@Nullable final T objectToCheck); + boolean meetsConditionFor(@Nullable T objectToCheck); } diff --git a/internal/utils/config/src/main/resources/ditto-devops.conf b/internal/utils/config/src/main/resources/ditto-devops.conf index 7679e1cb28d..d0fd0e1bac5 100644 --- a/internal/utils/config/src/main/resources/ditto-devops.conf +++ b/internal/utils/config/src/main/resources/ditto-devops.conf @@ -16,5 +16,9 @@ ditto.devops { // enables/disables the WoT (Web of Things) integration feature wot-integration-enabled = true wot-integration-enabled = ${?DITTO_DEVOPS_FEATURE_WOT_INTEGRATION_ENABLED} + + // enables/disables the historical API access feature + historical-apis-enabled = true + historical-apis-enabled = ${?DITTO_DEVOPS_FEATURE_HISTORICAL_APIS_ENABLED} } } diff --git a/internal/utils/config/src/main/resources/ditto-entity-creation.conf b/internal/utils/config/src/main/resources/ditto-entity-creation.conf index 9f9286739cb..2cd95bef807 100644 --- a/internal/utils/config/src/main/resources/ditto-entity-creation.conf +++ b/internal/utils/config/src/main/resources/ditto-entity-creation.conf @@ -2,7 +2,20 @@ ditto.entity-creation { # this default entry allows every authenticated "auth-subject" to create any "resource-type" in any "namespace": - grant = [{}] + grant = [ + { + resource-types = [ +// "policy" +// "thing" + ] + namespaces = [ +// "org.eclipse.ditto*" + ] + auth-subjects = [ +// "pre:ditto-*" + ] + } + ] # same as "grant", but rejecting requests which already passed "grant" revoke = [] } diff --git a/internal/utils/config/src/main/resources/ditto-kamon.conf b/internal/utils/config/src/main/resources/ditto-kamon.conf index 7462fa5381d..8cf38a24c97 100644 --- a/internal/utils/config/src/main/resources/ditto-kamon.conf +++ b/internal/utils/config/src/main/resources/ditto-kamon.conf @@ -72,6 +72,10 @@ kamon { trace { # disable reporting by default + # - always: report all traces. + # - never: don't report any trace. + # - random: randomly decide using the probability defined in the random-sampler.probability setting. + # - adaptive: keeps dynamic samplers for each operation while trying to achieve a set throughput goal. sampler = never sampler = ${?DITTO_TRACING_SAMPLER} diff --git a/internal/utils/config/src/main/resources/ditto-limits.conf b/internal/utils/config/src/main/resources/ditto-limits.conf index 1e8f1b20266..45a26d0f541 100644 --- a/internal/utils/config/src/main/resources/ditto-limits.conf +++ b/internal/utils/config/src/main/resources/ditto-limits.conf @@ -19,12 +19,6 @@ ditto.limits { messages { max-size = 250k max-size = ${?LIMITS_MESSAGES_MAX_SIZE} - - headers-size = 5k - headers-size = ${?LIMITS_MESSAGES_HEADERS_SIZE} - - auth-subjects-count = 100 - auth-subjects-count = ${?LIMITS_MESSAGES_AUTH_SUBJECTS_COUNT} } # limiations for the "things-search" service diff --git a/internal/utils/config/src/main/resources/ditto-protocol.conf b/internal/utils/config/src/main/resources/ditto-protocol.conf index 75fc78f98d8..a7714e1dd4b 100644 --- a/internal/utils/config/src/main/resources/ditto-protocol.conf +++ b/internal/utils/config/src/main/resources/ditto-protocol.conf @@ -5,5 +5,7 @@ ditto.protocol { "cache-control", "connection", "timeout-access", + "accept-encoding", + "x-forwarded-scheme", ] } diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/AbstractMongoEventAdapter.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/AbstractMongoEventAdapter.java index 75ea8d043c6..41e34a8462b 100644 --- a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/AbstractMongoEventAdapter.java +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/AbstractMongoEventAdapter.java @@ -12,20 +12,23 @@ */ package org.eclipse.ditto.internal.utils.persistence.mongo; +import java.util.Optional; import java.util.Set; -import javax.annotation.Nullable; - import org.bson.BsonDocument; import org.bson.BsonValue; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.DittoHeadersBuilder; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.events.Event; import org.eclipse.ditto.base.model.signals.events.EventRegistry; import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; +import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.json.JsonParseException; import org.eclipse.ditto.json.JsonValue; import org.slf4j.Logger; @@ -43,13 +46,21 @@ public abstract class AbstractMongoEventAdapter> implements E private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMongoEventAdapter.class); - @Nullable protected final ExtendedActorSystem system; + /** + * Internal header for persisting the historical headers for events. + */ + public static final JsonFieldDefinition HISTORICAL_EVENT_HEADERS = JsonFieldDefinition.ofJsonObject( + "__hh"); + + protected final ExtendedActorSystem system; protected final EventRegistry eventRegistry; + private final EventConfig eventConfig; - protected AbstractMongoEventAdapter(@Nullable final ExtendedActorSystem system, - final EventRegistry eventRegistry) { + protected AbstractMongoEventAdapter(final ExtendedActorSystem system, + final EventRegistry eventRegistry, final EventConfig eventConfig) { this.system = system; this.eventRegistry = eventRegistry; + this.eventConfig = eventConfig; } @Override @@ -66,9 +77,9 @@ public String manifest(final Object event) { public Object toJournal(final Object event) { if (event instanceof Event theEvent) { final JsonSchemaVersion schemaVersion = theEvent.getImplementedSchemaVersion(); - final JsonObject jsonObject = performToJournalMigration( + final JsonObject jsonObject = performToJournalMigration(theEvent, theEvent.toJson(schemaVersion, FieldType.regularOrSpecial()) - ); + ).build(); final BsonDocument bson = DittoBsonJson.getInstance().parse(jsonObject); final Set tags = theEvent.getDittoHeaders().getJournalTags(); return new Tagged(bson, tags); @@ -84,8 +95,12 @@ public EventSeq fromJournal(final Object event, final String manifest) { try { final JsonObject jsonObject = jsonValue.asObject() .setValue(EventsourcedEvent.JsonFields.REVISION.getPointer(), Event.DEFAULT_REVISION); + final DittoHeaders dittoHeaders = jsonObject.getValue(HISTORICAL_EVENT_HEADERS) + .map(obj -> DittoHeaders.newBuilder(obj).build()) + .orElse(DittoHeaders.empty()); + final T result = - eventRegistry.parse(performFromJournalMigration(jsonObject), DittoHeaders.empty()); + eventRegistry.parse(performFromJournalMigration(jsonObject), dittoHeaders); return EventSeq.single(result); } catch (final JsonParseException | DittoRuntimeException e) { if (system != null) { @@ -105,11 +120,22 @@ public EventSeq fromJournal(final Object event, final String manifest) { * Performs an optional migration of the passed in {@code jsonObject} (which is the JSON representation of the * {@link Event} to persist) just before it is transformed to Mongo BSON and inserted into the "journal" collection. * - * @param jsonObject the JsonObject representation of the {@link Event} to persist. + * @param event the event to apply journal migration for. + * @param jsonObject the JsonObject representation of the {@link org.eclipse.ditto.base.model.signals.events.Event} to persist. * @return the adjusted/migrated JsonObject to store. */ - protected JsonObject performToJournalMigration(final JsonObject jsonObject) { - return jsonObject; + protected JsonObjectBuilder performToJournalMigration(final Event event, final JsonObject jsonObject) { + return jsonObject.toBuilder() + .set(HISTORICAL_EVENT_HEADERS, calculateHistoricalHeaders(event.getDittoHeaders()).toJson()); + } + + private DittoHeaders calculateHistoricalHeaders(final DittoHeaders dittoHeaders) { + final DittoHeadersBuilder historicalHeadersBuilder = DittoHeaders.newBuilder(); + eventConfig.getHistoricalHeadersToPersist().forEach(headerKeyToPersist -> + Optional.ofNullable(dittoHeaders.get(headerKeyToPersist)) + .ifPresent(value -> historicalHeadersBuilder.putHeader(headerKeyToPersist, value)) + ); + return historicalHeadersBuilder.build(); } /** diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfig.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfig.java new file mode 100644 index 00000000000..0f053e50b96 --- /dev/null +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfig.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.internal.utils.persistence.mongo.config; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.internal.utils.config.ConfigWithFallback; +import org.eclipse.ditto.internal.utils.config.ScopedConfig; + +import com.typesafe.config.Config; + +/** + * This class implements the config for the handling of event journal entries. + */ +@Immutable +public final class DefaultEventConfig implements EventConfig { + + private static final String CONFIG_PATH = "event"; + + private final List historicalHeadersToPersist; + + private DefaultEventConfig(final ScopedConfig config) { + historicalHeadersToPersist = Collections.unmodifiableList(new ArrayList<>( + config.getStringList(EventConfigValue.HISTORICAL_HEADERS_TO_PERSIST.getConfigPath()) + )); + } + + /** + * Returns an instance of the default event journal config based on the settings of the specified Config. + * + * @param config is supposed to provide the settings of the event journal config at {@value #CONFIG_PATH}. + * @return instance + * @throws org.eclipse.ditto.internal.utils.config.DittoConfigError if {@code config} is invalid. + */ + public static DefaultEventConfig of(final Config config) { + return new DefaultEventConfig( + ConfigWithFallback.newInstance(config, CONFIG_PATH, EventConfigValue.values())); + } + + @Override + public List getHistoricalHeadersToPersist() { + return historicalHeadersToPersist; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DefaultEventConfig that = (DefaultEventConfig) o; + return Objects.equals(historicalHeadersToPersist, that.historicalHeadersToPersist); + } + + @Override + public int hashCode() { + return Objects.hash(historicalHeadersToPersist); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + + "historicalHeadersToPersist=" + historicalHeadersToPersist + + "]"; + } + +} diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultSnapshotConfig.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultSnapshotConfig.java index 40c25fd03af..e85967e7d93 100644 --- a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultSnapshotConfig.java +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultSnapshotConfig.java @@ -23,7 +23,7 @@ import com.typesafe.config.Config; /** - * This class implements the config for the handling of snapshots of policy entities. + * This class implements the config for the handling of snapshots of entities. */ @Immutable public final class DefaultSnapshotConfig implements SnapshotConfig { diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/EventConfig.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/EventConfig.java new file mode 100644 index 00000000000..436fdcc2286 --- /dev/null +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/EventConfig.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.internal.utils.persistence.mongo.config; + +import java.util.List; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; +import org.eclipse.ditto.internal.utils.config.KnownConfigValue; + +/** + * Provides configuration settings for the handling entity journal events. + */ +@Immutable +public interface EventConfig { + + /** + * Returns the DittoHeader keys to additionally persist for events in the event journal, e.g. in order + * to enable additional context/information for an audit log. + * + * @return the historical headers to persist into the event journal. + */ + List getHistoricalHeadersToPersist(); + + + /** + * An enumeration of the known config path expressions and their associated default values for + * {@code SnapshotConfig}. + */ + enum EventConfigValue implements KnownConfigValue { + + /** + * The DittoHeaders to persist when persisting events to the journal. + */ + HISTORICAL_HEADERS_TO_PERSIST("historical-headers-to-persist", List.of( + DittoHeaderDefinition.ORIGINATOR.getKey(), + DittoHeaderDefinition.CORRELATION_ID.getKey() + )); + + private final String path; + private final Object defaultValue; + + EventConfigValue(final String thePath, final Object theDefaultValue) { + path = thePath; + defaultValue = theDefaultValue; + } + + @Override + public Object getDefaultValue() { + return defaultValue; + } + + @Override + public String getConfigPath() { + return path; + } + + } + +} diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/MongoReadJournal.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/MongoReadJournal.java index b15afe15789..1d4d2ed8bab 100644 --- a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/MongoReadJournal.java +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/MongoReadJournal.java @@ -13,6 +13,7 @@ package org.eclipse.ditto.internal.utils.persistence.mongo.streaming; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -55,9 +56,19 @@ import akka.Done; import akka.NotUsed; import akka.actor.ActorSystem; +import akka.contrib.persistence.mongodb.JavaDslMongoReadJournal; import akka.contrib.persistence.mongodb.JournallingFieldNames$; import akka.contrib.persistence.mongodb.SnapshottingFieldNames$; import akka.japi.Pair; +import akka.persistence.query.EventEnvelope; +import akka.persistence.query.Offset; +import akka.persistence.query.PersistenceQuery; +import akka.persistence.query.javadsl.CurrentEventsByPersistenceIdQuery; +import akka.persistence.query.javadsl.CurrentEventsByTagQuery; +import akka.persistence.query.javadsl.CurrentPersistenceIdsQuery; +import akka.persistence.query.javadsl.EventsByPersistenceIdQuery; +import akka.persistence.query.javadsl.EventsByTagQuery; +import akka.persistence.query.javadsl.PersistenceIdsQuery; import akka.stream.Attributes; import akka.stream.Materializer; import akka.stream.RestartSettings; @@ -67,7 +78,7 @@ import akka.stream.javadsl.Source; /** - * Reads the event journal of com.github.scullxbones.akka-persistence-mongo plugin. + * Reads the event journal of {@code com.github.scullxbones.akka-persistence-mongo} plugin. * In the Akka system configuration, *
      *
    • @@ -81,12 +92,18 @@ *
    */ @AllValuesAreNonnullByDefault -public final class MongoReadJournal { +public final class MongoReadJournal implements CurrentEventsByPersistenceIdQuery, + CurrentEventsByTagQuery, CurrentPersistenceIdsQuery, EventsByPersistenceIdQuery, EventsByTagQuery, + PersistenceIdsQuery { /** - * ID field of documents delivered by the read journal. + * ID field of documents delivered by the journal collection. + */ + public static final String J_ID = JournallingFieldNames$.MODULE$.ID(); + + /** + * ID field of documents delivered by the snaps collection. */ - private static final String J_ID = JournallingFieldNames$.MODULE$.ID(); public static final String S_ID = J_ID; /** @@ -125,6 +142,11 @@ public final class MongoReadJournal { */ public static final String S_SN = SnapshottingFieldNames$.MODULE$.SEQUENCE_NUMBER(); + /** + * Document field of the timestamp of snapshots. + */ + public static final String S_TS = SnapshottingFieldNames$.MODULE$.TIMESTAMP(); + private static final String S_SERIALIZED_SNAPSHOT = "s2"; /** @@ -147,8 +169,11 @@ public final class MongoReadJournal { private final DittoMongoClient mongoClient; private final IndexInitializer indexInitializer; + private final JavaDslMongoReadJournal akkaReadJournal; + private MongoReadJournal(final String journalCollection, final String snapsCollection, + final String readJournalConfigurationKey, final DittoMongoClient mongoClient, final ActorSystem actorSystem) { @@ -157,6 +182,8 @@ private MongoReadJournal(final String journalCollection, this.mongoClient = mongoClient; final var materializer = SystemMaterializer.get(actorSystem).materializer(); indexInitializer = IndexInitializer.of(mongoClient.getDefaultDatabase(), materializer); + akkaReadJournal = PersistenceQuery.get(actorSystem) + .getReadJournalFor(JavaDslMongoReadJournal.class, readJournalConfigurationKey); } /** @@ -188,7 +215,12 @@ public static MongoReadJournal newInstance(final Config config, final DittoMongo getOverrideCollectionName(config.getConfig(autoStartJournalKey), JOURNAL_COLLECTION_NAME_KEY); final String snapshotCollection = getOverrideCollectionName(config.getConfig(autoStartSnapsKey), SNAPS_COLLECTION_NAME_KEY); - return new MongoReadJournal(journalCollection, snapshotCollection, mongoClient, actorSystem); + return new MongoReadJournal(journalCollection, + snapshotCollection, + autoStartJournalKey + "-read", + mongoClient, + actorSystem + ); } /** @@ -397,6 +429,29 @@ public Source getJournalPidsAboveWithTag(final String lowerBoun .mapConcat(pids -> pids); } + /** + * A Source retrieving a single revision/sequence number of type {@code long} for the last snapshot sequence number + * available for the passed {@code pid} and before the passed {@code timestamp}. + * + * @param pid the persistenceId to find out the last snapshot sequence number for. + * @param timestamp the timestamp to use as selection criteria for the snapshot sequence number to find out. + * @return a Source of a single element with the determined snapshot sequence number. + */ + public Source getLastSnapshotSequenceNumberBeforeTimestamp(final String pid, + final Instant timestamp) { + + final Bson filter = Filters.and( + Filters.eq(S_PROCESSOR_ID, pid), + Filters.lte(S_TS, timestamp.toEpochMilli()) + ); + return getSnapshotStore().flatMapConcat(snaps -> Source.fromPublisher(snaps + .find(filter) + .projection(Projections.include(S_SN)) + .sort(Sorts.descending(S_SN)) + .first() + )).map(document -> document.getLong(S_SN)); + } + /** * Retrieve all latest snapshots with unique PIDs in snapshot store above a lower bound. * Does not limit database access in any way. @@ -412,7 +467,7 @@ public Source getNewestSnapshotsAbove(final String lowerBound final Materializer mat, final String... snapshotFields) { - return getNewestSnapshotsAbove(lowerBoundPid, batchSize, false, mat, snapshotFields); + return getNewestSnapshotsAbove(lowerBoundPid, batchSize, false, Duration.ZERO, mat, snapshotFields); } /** @@ -422,6 +477,8 @@ public Source getNewestSnapshotsAbove(final String lowerBound * @param lowerBoundPid the lower-bound PID. * @param batchSize how many snapshots to read in 1 query. * @param includeDeleted whether to include deleted snapshots. + * @param minAgeFromNow the minimum age (based on {@code Instant.now()}) the snapshot must have in order to get + * selected. * @param mat the materializer. * @param snapshotFields snapshot fields to project out. * @return source of newest snapshots with unique PIDs. @@ -429,14 +486,20 @@ public Source getNewestSnapshotsAbove(final String lowerBound public Source getNewestSnapshotsAbove(final String lowerBoundPid, final int batchSize, final boolean includeDeleted, + final Duration minAgeFromNow, final Materializer mat, final String... snapshotFields) { return getSnapshotStore() .withAttributes(Attributes.inputBuffer(1, 1)) .flatMapConcat(snapshotStore -> - listNewestSnapshots(snapshotStore, SnapshotFilter.of(lowerBoundPid), batchSize, includeDeleted, mat, - snapshotFields) + listNewestSnapshots(snapshotStore, + SnapshotFilter.of(lowerBoundPid, minAgeFromNow), + batchSize, + includeDeleted, + mat, + snapshotFields + ) ) .mapConcat(pids -> pids); } @@ -460,8 +523,7 @@ public Source getNewestSnapshotsAbove( return getSnapshotStore() .withAttributes(Attributes.inputBuffer(1, 1)) .flatMapConcat(snapshotStore -> - listNewestSnapshots(snapshotStore, snapshotFilter, batchSize, false, mat, - snapshotFields) + listNewestSnapshots(snapshotStore, snapshotFilter, batchSize, false, mat, snapshotFields) ) .mapConcat(pids -> pids); } @@ -509,12 +571,12 @@ public Source, NotUsed> getSmallestSnapshotSeqNo(final String pid * @return source of the delete result. */ public Source deleteEvents(final String pid, final long minSeqNr, final long maxSeqNr) { + + final Bson filter = Filters.and(Filters.eq(J_PROCESSOR_ID, pid), + Filters.gte(J_TO, minSeqNr), + Filters.lte(J_TO, maxSeqNr)); return getJournal() - .flatMapConcat(journal -> Source.fromPublisher( - journal.deleteMany(Filters.and(Filters.eq(J_PROCESSOR_ID, pid), - Filters.gte(J_TO, minSeqNr), - Filters.lte(J_TO, maxSeqNr))) - )); + .flatMapConcat(journal -> Source.fromPublisher(journal.deleteMany(filter))); } /** @@ -526,12 +588,46 @@ public Source deleteEvents(final String pid, final long m * @return source of the delete result. */ public Source deleteSnapshots(final String pid, final long minSeqNr, final long maxSeqNr) { + + final Bson filter = Filters.and(Filters.eq(S_PROCESSOR_ID, pid), + Filters.gte(S_SN, minSeqNr), + Filters.lte(S_SN, maxSeqNr)); return getSnapshotStore() - .flatMapConcat(snaps -> Source.fromPublisher( - snaps.deleteMany(Filters.and(Filters.eq(S_PROCESSOR_ID, pid), - Filters.gte(S_SN, minSeqNr), - Filters.lte(S_SN, maxSeqNr))) - )); + .flatMapConcat(snaps -> Source.fromPublisher(snaps.deleteMany(filter))); + } + + + @Override + public Source currentEventsByPersistenceId(final String persistenceId, + final long fromSequenceNr, + final long toSequenceNr) { + return akkaReadJournal.currentEventsByPersistenceId(persistenceId, fromSequenceNr, toSequenceNr); + } + + @Override + public Source currentEventsByTag(final String tag, final Offset offset) { + return akkaReadJournal.currentEventsByTag(tag, offset); + } + + @Override + public Source currentPersistenceIds() { + return akkaReadJournal.currentPersistenceIds(); + } + + @Override + public Source eventsByPersistenceId(final String persistenceId, final long fromSequenceNr, + final long toSequenceNr) { + return akkaReadJournal.eventsByPersistenceId(persistenceId, fromSequenceNr, toSequenceNr); + } + + @Override + public Source eventsByTag(final String tag, final Offset offset) { + return akkaReadJournal.eventsByTag(tag, offset); + } + + @Override + public Source persistenceIds() { + return akkaReadJournal.persistenceIds(); } private Source, NotUsed> listPidsInJournal(final MongoCollection journal, @@ -570,11 +666,16 @@ private Source, NotUsed> listNewestSnapshots(final MongoCollectio final Materializer mat, final String... snapshotFields) { - return unfoldBatchedSource(filter.getLowerBoundPid(), + return unfoldBatchedSource(filter.lowerBoundPid(), mat, SnapshotBatch::maxPid, - actualStartPid -> listNewestActiveSnapshotsByBatch(snapshotStore, filter.withLowerBound(actualStartPid), batchSize, - includeDeleted, snapshotFields)) + actualStartPid -> listNewestActiveSnapshotsByBatch(snapshotStore, + filter.withLowerBound(actualStartPid), + batchSize, + includeDeleted, + snapshotFields + ) + ) .mapConcat(x -> x) .map(SnapshotBatch::items); } @@ -776,8 +877,8 @@ private static Source listNewestActiveSnapshotsByBatch( final String... snapshotFields) { final List pipeline = new ArrayList<>(5); - // optional match stage - snapshotFilter.toMongoFilter().ifPresent(bson -> pipeline.add(Aggregates.match(bson))); + // match stage + pipeline.add(Aggregates.match(snapshotFilter.toMongoFilter())); // sort stage pipeline.add(Aggregates.sort(Sorts.orderBy(Sorts.ascending(S_PROCESSOR_ID), Sorts.descending(S_SN)))); @@ -817,7 +918,9 @@ private static Source listNewestActiveSnapshotsByBatch( if (theMaxPid == null) { return Source.empty(); } else { - return Source.single(new SnapshotBatch(theMaxPid, document.getList(items, Document.class))); + final SnapshotBatch snapshotBatch = + new SnapshotBatch(theMaxPid, document.getList(items, Document.class)); + return Source.single(snapshotBatch); } }); } diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/SnapshotFilter.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/SnapshotFilter.java index b5519c0c157..d1cc891adb7 100644 --- a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/SnapshotFilter.java +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/SnapshotFilter.java @@ -13,9 +13,12 @@ package org.eclipse.ditto.internal.utils.persistence.mongo.streaming; -import java.util.Optional; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; import org.bson.conversions.Bson; +import org.bson.types.ObjectId; import com.mongodb.client.model.Filters; @@ -23,16 +26,27 @@ /** * A record that hold optional filters for retrieving snapshots from persistence. + * + * @param lowerBoundPid the lower-bound pid from which to start reading the snapshots + * @param pidFilter the regex applied to the pid to filter the snapshots + * @param minAgeFromNow the minimum age (based on {@code Instant.now()}) the snapshot must have in order to get + * selected */ -public record SnapshotFilter(String lowerBoundPid, String pidFilter) { +public record SnapshotFilter(String lowerBoundPid, String pidFilter, Duration minAgeFromNow) { /** * Document field of PID in snapshot stores. */ private static final String S_PROCESSOR_ID = SnapshottingFieldNames$.MODULE$.PROCESSOR_ID(); - static SnapshotFilter of(final String lowerBoundPid) { - return new SnapshotFilter(lowerBoundPid, ""); + /** + * @param lowerBoundPid the lower-bound pid from which to start reading the snapshots + * @param minAgeFromNow the minimum age (based on {@code Instant.now()}) the snapshot must have in order to get + * selected. + * @return new instance of SnapshotFilter + */ + static SnapshotFilter of(final String lowerBoundPid, final Duration minAgeFromNow) { + return of(lowerBoundPid, "", minAgeFromNow); } /** @@ -41,14 +55,19 @@ static SnapshotFilter of(final String lowerBoundPid) { * @return new instance of SnapshotFilter */ public static SnapshotFilter of(final String lowerBoundPid, final String pidFilter) { - return new SnapshotFilter(lowerBoundPid, pidFilter); + return of(lowerBoundPid, pidFilter, Duration.ZERO); } /** - * @return the lower-bound pid + * @param lowerBoundPid the lower-bound pid from which to start reading the snapshots + * @param pidFilter the regex applied to the pid to filter the snapshots + * @param minAgeFromNow the minimum age (based on {@code Instant.now()}) the snapshot must have in order to get + * selected. + * @return new instance of SnapshotFilter */ - String getLowerBoundPid() { - return lowerBoundPid; + public static SnapshotFilter of(final String lowerBoundPid, final String pidFilter, + final Duration minAgeFromNow) { + return new SnapshotFilter(lowerBoundPid, pidFilter, minAgeFromNow); } /** @@ -56,21 +75,32 @@ String getLowerBoundPid() { * @return a new instance of SnapshotFilter with the new lower-bound pid set */ SnapshotFilter withLowerBound(final String newLowerBoundPid) { - return new SnapshotFilter(newLowerBoundPid, pidFilter); + return new SnapshotFilter(newLowerBoundPid, pidFilter, minAgeFromNow); } /** * @return a Bson filter that can be used in a mongo query to filter the snapshots or an empty Optional if no filter was set */ - Optional toMongoFilter() { + Bson toMongoFilter() { + final Bson filter; if (!lowerBoundPid.isEmpty() && !pidFilter.isEmpty()) { - return Optional.of(Filters.and(getLowerBoundFilter(), getNamespacesFilter())); + filter = Filters.and(getLowerBoundFilter(), getNamespacesFilter()); } else if (!lowerBoundPid.isEmpty()) { - return Optional.of(getLowerBoundFilter()); + filter = getLowerBoundFilter(); } else if (!pidFilter.isEmpty()) { - return Optional.of(getNamespacesFilter()); + filter = getNamespacesFilter(); + } else { + filter = Filters.empty(); + } + + if (minAgeFromNow.isZero()) { + return filter; } else { - return Optional.empty(); + final Date nowMinusMinAgeFromNow = Date.from(Instant.now().minus(minAgeFromNow)); + final Bson eventRetentionFilter = Filters.lt("_id", + ObjectId.getSmallestWithDate(nowMinusMinAgeFromNow) + ); + return Filters.and(filter, eventRetentionFilter); } } diff --git a/internal/utils/persistence/src/test/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfigTest.java b/internal/utils/persistence/src/test/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfigTest.java new file mode 100644 index 00000000000..97618879b77 --- /dev/null +++ b/internal/utils/persistence/src/test/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfigTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.internal.utils.persistence.mongo.config; + +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.List; + +import org.assertj.core.api.JUnitSoftAssertions; +import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Unit test for {@link DefaultEventConfig}. + */ +public final class DefaultEventConfigTest { + + private static Config snapshotTestConf; + + @Rule + public final JUnitSoftAssertions softly = new JUnitSoftAssertions(); + + @BeforeClass + public static void initTestFixture() { + snapshotTestConf = ConfigFactory.load("event-test"); + } + + @Test + public void assertImmutability() { + assertInstancesOf(DefaultEventConfig.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(DefaultEventConfig.class) + .usingGetClass() + .verify(); + } + + @Test + public void underTestReturnsDefaultValuesIfBaseConfigWasEmpty() { + final DefaultEventConfig underTest = DefaultEventConfig.of(ConfigFactory.empty()); + + softly.assertThat(underTest.getHistoricalHeadersToPersist()) + .as(EventConfig.EventConfigValue.HISTORICAL_HEADERS_TO_PERSIST.getConfigPath()) + .isEqualTo(EventConfig.EventConfigValue.HISTORICAL_HEADERS_TO_PERSIST.getDefaultValue()); + } + + @Test + public void underTestReturnsValuesOfConfigFile() { + final DefaultEventConfig underTest = DefaultEventConfig.of(snapshotTestConf); + + softly.assertThat(underTest.getHistoricalHeadersToPersist()) + .as(EventConfig.EventConfigValue.HISTORICAL_HEADERS_TO_PERSIST.getConfigPath()) + .isEqualTo(List.of(DittoHeaderDefinition.ORIGINATOR.getKey(), "foo")); + } +} diff --git a/internal/utils/persistence/src/test/resources/event-test.conf b/internal/utils/persistence/src/test/resources/event-test.conf new file mode 100644 index 00000000000..3b84b50ef7a --- /dev/null +++ b/internal/utils/persistence/src/test/resources/event-test.conf @@ -0,0 +1,6 @@ +event { + historical-headers-to-persist = [ + "ditto-originator" + "foo" + ] +} diff --git a/internal/utils/persistence/src/test/resources/mongo-read-journal-test.conf b/internal/utils/persistence/src/test/resources/mongo-read-journal-test.conf index c1a2b8663ca..1905bbf9f7e 100644 --- a/internal/utils/persistence/src/test/resources/mongo-read-journal-test.conf +++ b/internal/utils/persistence/src/test/resources/mongo-read-journal-test.conf @@ -32,6 +32,17 @@ akka-contrib-mongodb-persistence-test-journal { } } +akka-contrib-mongodb-persistence-test-journal-read { + class = "akka.contrib.persistence.mongodb.MongoReadJournal" + + overrides { + journal-collection = "test_journal" + journal-index = "test_journal_index" + realtime-collection = "test_realtime" + metadata-collection = "test_metadata" + } +} + akka-contrib-mongodb-persistence-test-snapshots { class = "akka.contrib.persistence.mongodb.MongoSnapshots" diff --git a/internal/utils/persistence/src/test/resources/test.conf b/internal/utils/persistence/src/test/resources/test.conf index a975197c7fa..cd714bab1ac 100644 --- a/internal/utils/persistence/src/test/resources/test.conf +++ b/internal/utils/persistence/src/test/resources/test.conf @@ -132,6 +132,17 @@ akka-contrib-mongodb-persistence-test-journal { } } +akka-contrib-mongodb-persistence-test-journal-read { + class = "akka.contrib.persistence.mongodb.MongoReadJournal" + + overrides { + journal-collection = "test_journal" + journal-index = "test_journal_index" + realtime-collection = "test_realtime" + metadata-collection = "test_metadata" + } +} + akka-contrib-mongodb-persistence-test-snapshots { class = "akka.persistence.inmemory.snapshot.InMemorySnapshotStore" diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceActor.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceActor.java index 5e0746b82cd..d4f5e9cd6aa 100755 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceActor.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceActor.java @@ -13,30 +13,44 @@ package org.eclipse.ditto.internal.utils.persistentactors; import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; import java.util.function.Consumer; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; +import org.bson.BsonDocument; import org.eclipse.ditto.base.api.commands.sudo.SudoCommand; import org.eclipse.ditto.base.model.entity.id.EntityId; import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.DittoHeadersSettable; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.json.Jsonifiable; +import org.eclipse.ditto.base.model.signals.FeatureToggle; import org.eclipse.ditto.base.model.signals.commands.Command; -import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; +import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry; import org.eclipse.ditto.internal.utils.akka.PingCommand; import org.eclipse.ditto.internal.utils.akka.PingCommandResponse; +import org.eclipse.ditto.internal.utils.akka.logging.DittoDiagnosticLoggingAdapter; import org.eclipse.ditto.internal.utils.config.ScopedConfig; import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces; import org.eclipse.ditto.internal.utils.persistence.SnapshotAdapter; +import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; +import org.eclipse.ditto.internal.utils.persistence.mongo.DittoBsonJson; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.commands.CommandStrategy; import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.internal.utils.persistentactors.results.Result; @@ -45,16 +59,24 @@ import org.eclipse.ditto.internal.utils.tracing.DittoTracing; import org.eclipse.ditto.internal.utils.tracing.span.SpanOperationName; import org.eclipse.ditto.internal.utils.tracing.span.SpanTagKey; +import org.eclipse.ditto.internal.utils.tracing.span.StartedSpan; import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonValue; import akka.actor.ActorRef; +import akka.actor.Cancellable; import akka.japi.pf.ReceiveBuilder; +import akka.pattern.Patterns; import akka.persistence.RecoveryCompleted; import akka.persistence.RecoveryTimedOut; import akka.persistence.SaveSnapshotFailure; import akka.persistence.SaveSnapshotSuccess; import akka.persistence.SnapshotOffer; +import akka.persistence.SnapshotProtocol; +import akka.persistence.SnapshotSelectionCriteria; +import akka.persistence.query.EventEnvelope; +import akka.stream.javadsl.Sink; import scala.Option; /** @@ -72,7 +94,8 @@ public abstract class AbstractPersistenceActor< S extends Jsonifiable.WithFieldSelectorAndPredicate, I extends EntityId, K, - E extends Event> extends AbstractPersistentActorWithTimersAndCleanup implements ResultVisitor { + E extends EventsourcedEvent> + extends AbstractPersistentActorWithTimersAndCleanup implements ResultVisitor { /** * An event journal {@code Tag} used to tag journal entries managed by a PersistenceActor as "always alive" meaning @@ -83,6 +106,7 @@ public abstract class AbstractPersistenceActor< private final SnapshotAdapter snapshotAdapter; private final Receive handleEvents; private final Receive handleCleanups; + private final MongoReadJournal mongoReadJournal; private long lastSnapshotRevision; private long confirmedSnapshotRevision; @@ -104,10 +128,12 @@ public abstract class AbstractPersistenceActor< * Instantiate the actor. * * @param entityId the entity ID. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the entity. */ @SuppressWarnings("unchecked") - protected AbstractPersistenceActor(final I entityId) { + protected AbstractPersistenceActor(final I entityId, final MongoReadJournal mongoReadJournal) { this.entityId = entityId; + this.mongoReadJournal = mongoReadJournal; final var actorSystem = context().system(); final var dittoExtensionsConfig = ScopedConfig.dittoExtension(actorSystem.settings().config()); this.snapshotAdapter = SnapshotAdapter.get(actorSystem, dittoExtensionsConfig); @@ -190,6 +216,20 @@ protected void onEntityModified() { */ protected abstract DittoRuntimeExceptionBuilder newNotAccessibleExceptionBuilder(); + /** + * @param revision the revision which could not be resolved in the entity history. + * @return An exception builder to respond to unexpected commands addressed to a nonexistent historical entity at a + * given {@code revision}. + */ + protected abstract DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(long revision); + + /** + * @param timestamp the timestamp which could not be resolved in the entity history. + * @return An exception builder to respond to unexpected commands addressed to a nonexistent historical entity at a + * given {@code timestamp}. + */ + protected abstract DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(Instant timestamp); + /** * Publish an event. * @@ -292,6 +332,8 @@ protected void becomeCreatedHandler() { final CommandStrategy commandStrategy = getCreatedStrategy(); final Receive receive = handleCleanups.orElse(ReceiveBuilder.create() + .match(commandStrategy.getMatchingClass(), this::isHistoricalRetrieveCommand, + this::handleHistoricalRetrieveCommand) .match(commandStrategy.getMatchingClass(), commandStrategy::isDefined, this::handleByCommandStrategy) .match(PersistEmptyEvent.class, this::handlePersistEmptyEvent) .match(CheckForActivity.class, this::checkForActivity) @@ -299,6 +341,8 @@ protected void becomeCreatedHandler() { .matchEquals(Control.TAKE_SNAPSHOT, this::takeSnapshotByInterval) .match(SaveSnapshotSuccess.class, this::saveSnapshotSuccess) .match(SaveSnapshotFailure.class, this::saveSnapshotFailure) + .match(PersistEventAsync.class, persistEventAsync -> + persistAndApplyEvent((E) persistEventAsync.event, persistEventAsync.handler)) .build()) .orElse(matchAnyAfterInitialization()); @@ -308,6 +352,145 @@ protected void becomeCreatedHandler() { scheduleSnapshot(); } + private boolean isHistoricalRetrieveCommand(final C command) { + final DittoHeaders headers = command.getDittoHeaders(); + return command.getCategory().equals(Command.Category.QUERY) && ( + headers.containsKey(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey()) || + headers.containsKey(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey()) + ); + } + + private void handleHistoricalRetrieveCommand(final C command) { + + try { + FeatureToggle.checkHistoricalApiAccessFeatureEnabled(command.getType(), command.getDittoHeaders()); + } catch (final DittoRuntimeException dre) { + getSender().tell(dre, getSelf()); + return; + } + + final CommandStrategy commandStrategy = getCreatedStrategy(); + final EventStrategy eventStrategy = getEventStrategy(); + final ActorRef sender = getSender(); + final ActorRef self = getSelf(); + final long atHistoricalRevision = Optional + .ofNullable(command.getDittoHeaders().get(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey())) + .map(Long::parseLong) + .orElseGet(this::lastSequenceNr); + final Instant atHistoricalTimestamp = Optional + .ofNullable(command.getDittoHeaders().get(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey())) + .map(Instant::parse) + .orElse(Instant.EPOCH); + + loadSnapshot(persistenceId(), SnapshotSelectionCriteria.create( + atHistoricalRevision, + atHistoricalTimestamp.equals(Instant.EPOCH) ? Long.MAX_VALUE : atHistoricalTimestamp.toEpochMilli(), + 0L, + 0L + ), getLatestSnapshotSequenceNumber()); + + final Duration waitTimeout = Duration.ofSeconds(5); + final Cancellable cancellableSnapshotLoadTimeout = + getContext().getSystem().getScheduler().scheduleOnce(waitTimeout, getSelf(), waitTimeout, + getContext().getDispatcher(), getSelf()); + getContext().become(ReceiveBuilder.create() + .match(SnapshotProtocol.LoadSnapshotResult.class, loadSnapshotResult -> + historicalRetrieveHandleLoadSnapshotResult(command, + commandStrategy, + eventStrategy, + sender, + self, + atHistoricalRevision, + atHistoricalTimestamp, + cancellableSnapshotLoadTimeout, + loadSnapshotResult + ) + ) + .match(SnapshotProtocol.LoadSnapshotFailed.class, loadSnapshotFailed -> + log.warning(loadSnapshotFailed.cause(), "Loading snapshot failed") + ) + .matchEquals(waitTimeout, wt -> { + log.withCorrelationId(command) + .warning("Timed out waiting for receiving snapshot result!"); + becomeCreatedOrDeletedHandler(); + unstashAll(); + }) + .matchAny(any -> stash()) + .build()); + } + + private void historicalRetrieveHandleLoadSnapshotResult(final C command, + final CommandStrategy commandStrategy, + final EventStrategy eventStrategy, + final ActorRef sender, + final ActorRef self, + final long atHistoricalRevision, + final Instant atHistoricalTimestamp, + final Cancellable cancellableSnapshotLoadTimeout, + final SnapshotProtocol.LoadSnapshotResult loadSnapshotResult) { + + final Option snapshotEntity = loadSnapshotResult.snapshot() + .map(snapshotAdapter::fromSnapshotStore); + final boolean snapshotIsPresent = snapshotEntity.isDefined(); + + if (snapshotIsPresent || getLatestSnapshotSequenceNumber() == 0) { + final long snapshotEntityRevision = snapshotIsPresent ? + loadSnapshotResult.snapshot().get().metadata().sequenceNr() : 0L; + + final long fromSequenceNr; + if (atHistoricalRevision == snapshotEntityRevision) { + fromSequenceNr = snapshotEntityRevision; + } else { + fromSequenceNr = snapshotEntityRevision + 1; + } + + @Nullable final S entityFromSnapshot = snapshotIsPresent ? snapshotEntity.get() : null; + mongoReadJournal.currentEventsByPersistenceId(persistenceId(), + fromSequenceNr, + atHistoricalRevision + ) + .map(AbstractPersistenceActor::mapJournalEntryToEvent) + .map(journalEntryEvent -> new EntityWithEvent( + eventStrategy.handle((E) journalEntryEvent, entityFromSnapshot, journalEntryEvent.getRevision()), + (E) journalEntryEvent + )) + .takeWhile(entityWithEvent -> { + if (atHistoricalTimestamp.equals(Instant.EPOCH)) { + // no at-historical-timestamp was specified, so take all up to "at-historical-revision": + return true; + } else { + // take while the timestamps of the events are before the specified "at-historical-timestamp": + return entityWithEvent.event.getTimestamp() + .filter(ts -> ts.isBefore(atHistoricalTimestamp)) + .isPresent(); + } + }) + .reduce((ewe1, ewe2) -> new EntityWithEvent( + eventStrategy.handle(ewe2.event, ewe2.entity, ewe2.revision), + ewe2.event + )) + .runWith(Sink.foreach(entityWithEvent -> + commandStrategy.apply(getStrategyContext(), + entityWithEvent.entity, + entityWithEvent.revision, + command + ).accept(new HistoricalResultListener(sender, + entityWithEvent.event.getDittoHeaders())) + ), + getContext().getSystem()); + } else { + if (!atHistoricalTimestamp.equals(Instant.EPOCH)) { + sender.tell(newHistoryNotAccessibleExceptionBuilder(atHistoricalTimestamp).build(), self); + } else { + sender.tell(newHistoryNotAccessibleExceptionBuilder(atHistoricalRevision).build(), self); + } + } + + cancellableSnapshotLoadTimeout.cancel(); + becomeCreatedOrDeletedHandler(); + unstashAll(); + } + /** * Processes a received {@link PingCommand}. * May be overwritten in order to hook into processing ping commands with additional functionality. @@ -357,8 +540,23 @@ protected void persistAndApplyEvent(final E event, final BiConsumer handle } } + private record PersistEventAsync< + E extends EventsourcedEvent, + S extends Jsonifiable.WithFieldSelectorAndPredicate>(E event, BiConsumer handler) {}; + + /** + * Persist an event, modify actor state by the event strategy, then invoke the handler. + * + * @param event the event to persist and apply. + * @param handler what happens afterwards. + */ + protected void persistAndApplyEventAsync(final CompletionStage event, final BiConsumer handler) { + Patterns.pipe(event.thenApply(e -> new PersistEventAsync<>(e, handler)), getContext().getDispatcher()) + .to(getSelf()); + } + /** - * Allows to modify the passed in {@code event} before {@link #persistEvent(Event, Consumer)} is invoked. + * Allows to modify the passed in {@code event} before {@link #persistEvent(EventsourcedEvent, Consumer)} is invoked. * Overwrite this method and call the super method in order to additionally modify the event before persisting it. * * @param event the event to potentially modify. @@ -424,6 +622,8 @@ private Receive createDeletedBehavior() { .matchEquals(Control.TAKE_SNAPSHOT, this::takeSnapshotByInterval) .match(SaveSnapshotSuccess.class, this::saveSnapshotSuccess) .match(SaveSnapshotFailure.class, this::saveSnapshotFailure) + .match(PersistEventAsync.class, persistEventAsync -> + persistAndApplyEvent((E) persistEventAsync.event, persistEventAsync.handler)) .build()) .orElse(matchAnyWhenDeleted()); } @@ -452,7 +652,7 @@ private void cancelSnapshot() { } protected void handleByCommandStrategy(final C command) { - handleByStrategy(command, getCreatedStrategy()); + handleByStrategy(command, entity, getCreatedStrategy()); } @SuppressWarnings("unchecked") @@ -462,11 +662,12 @@ private ReceiveBuilder handleByDeletedStrategyReceiveBuilder() { .match(deletedStrategy.getMatchingClass(), deletedStrategy::isDefined, // get the current deletedStrategy during "matching time" to allow implementing classes // to update the strategy during runtime - command -> handleByStrategy(command, (CommandStrategy) getDeletedStrategy())); + command -> handleByStrategy(command, entity, (CommandStrategy) getDeletedStrategy())); } @SuppressWarnings("unchecked") - private > void handleByStrategy(final T command, final CommandStrategy strategy) { + private > void handleByStrategy(final T command, @Nullable final S workEntity, + final CommandStrategy strategy) { log.debug("Handling by strategy: <{}>", command); final var startedSpan = DittoTracing.newPreparedSpan( @@ -481,11 +682,16 @@ private > void handleByStrategy(final T command, final Comm accessCounter++; Result result; try { - result = strategy.apply(getStrategyContext(), entity, getNextRevisionNumber(), (T) tracedCommand); + result = strategy.apply(getStrategyContext(), workEntity, getNextRevisionNumber(), (T) tracedCommand); result.accept(this); - } catch (final DittoRuntimeException e) { + } catch (final CompletionException | DittoRuntimeException e) { + final DittoRuntimeException dittoRuntimeException = + DittoRuntimeException.asDittoRuntimeException(e, throwable -> + DittoInternalErrorException.newBuilder() + .dittoHeaders(command.getDittoHeaders()) + .build()); startedSpan.tagAsFailed(e); - result = ResultFactory.newErrorResult(e, tracedCommand); + result = ResultFactory.newErrorResult(dittoRuntimeException, tracedCommand); result.accept(this); } finally { startedSpan.finish(); @@ -495,11 +701,31 @@ private > void handleByStrategy(final T command, final Comm @Override public void onMutation(final Command command, final E event, final WithDittoHeaders response, - final boolean becomeCreated, final boolean becomeDeleted) { + final boolean becomeCreated, final boolean becomeDeleted) { + final ActorRef sender = getSender(); persistAndApplyEvent(event, (persistedEvent, resultingEntity) -> { if (shouldSendResponse(command.getDittoHeaders())) { - notifySender(response); + notifySender(sender, response); + } + if (becomeDeleted) { + becomeDeletedHandler(); + } + if (becomeCreated) { + becomeCreatedHandler(); + } + }); + } + + @Override + public void onStagedMutation(final Command command, final CompletionStage event, + final CompletionStage response, + final boolean becomeCreated, final boolean becomeDeleted) { + + final ActorRef sender = getSender(); + persistAndApplyEventAsync(event, (persistedEvent, resultingEntity) -> { + if (shouldSendResponse(command.getDittoHeaders())) { + notifySender(sender, response); } if (becomeDeleted) { becomeDeletedHandler(); @@ -513,7 +739,16 @@ public void onMutation(final Command command, final E event, final WithDittoH @Override public void onQuery(final Command command, final WithDittoHeaders response) { if (command.getDittoHeaders().isResponseRequired()) { - notifySender(response); + final ActorRef sender = getSender(); + notifySender(sender, response); + } + } + + @Override + public void onStagedQuery(final Command command, final CompletionStage response) { + if (command.getDittoHeaders().isResponseRequired()) { + final ActorRef sender = getSender(); + response.thenAccept(r -> notifySender(sender, r)); } } @@ -553,28 +788,31 @@ private void persistEvent(final E event, final Consumer handler) { persist( event.setDittoHeaders(DittoHeaders.of(persistOperationSpan.propagateContext(event.getDittoHeaders()))), - persistedEvent -> { - l.info("Successfully persisted Event <{}> w/ rev: <{}>.", - persistedEvent.getType(), - getRevisionNumber()); - persistOperationSpan.finish(); - - /* - * The event has to be applied before creating the snapshot, otherwise a snapshot with new - * sequence no (e.g. 2), but old entity revision no (e.g. 1) will be created -> can lead to serious - * aftereffects. - */ - handler.accept(persistedEvent); - onEntityModified(); - - // save a snapshot if there were too many changes since the last snapshot - if (snapshotThresholdPassed()) { - takeSnapshot("snapshot threshold is reached"); - } - } + persistedEvent -> handlePersistedEvent(handler, l, persistOperationSpan, persistedEvent) ); } + private void handlePersistedEvent(final Consumer handler, final DittoDiagnosticLoggingAdapter l, + final StartedSpan persistOperationSpan, final E persistedEvent) { + l.info("Successfully persisted Event <{}> w/ rev: <{}>.", + persistedEvent.getType(), + getRevisionNumber()); + persistOperationSpan.finish(); + + /* + * The event has to be applied before creating the snapshot, otherwise a snapshot with new + * sequence no (e.g. 2), but old entity revision no (e.g. 1) will be created -> can lead to serious + * aftereffects. + */ + handler.accept(persistedEvent); + onEntityModified(); + + // save a snapshot if there were too many changes since the last snapshot + if (snapshotThresholdPassed()) { + takeSnapshot("snapshot threshold is reached"); + } + } + private void takeSnapshot(final String reason) { if (entityId instanceof NamespacedEntityId namespacedEntityId) { @@ -626,6 +864,10 @@ private void notifySender(final WithDittoHeaders message) { notifySender(getSender(), message); } + private void notifySender(final ActorRef sender, final CompletionStage message) { + message.thenAccept(msg -> notifySender(sender, msg)); + } + private void takeSnapshotByInterval(final Control takeSnapshot) { takeSnapshot("snapshot interval has passed"); } @@ -713,6 +955,18 @@ public static Object checkForActivity(final long accessCounter) { return new CheckForActivity(accessCounter); } + private static EventsourcedEvent mapJournalEntryToEvent(final EventEnvelope eventEnvelope) { + + final BsonDocument event = (BsonDocument) eventEnvelope.event(); + final JsonObject eventAsJsonObject = DittoBsonJson.getInstance() + .serialize(event); + + final DittoHeaders dittoHeaders = eventAsJsonObject.getValue(AbstractMongoEventAdapter.HISTORICAL_EVENT_HEADERS) + .map(obj -> DittoHeaders.newBuilder(obj).build()) + .orElseGet(DittoHeaders::empty); + return (EventsourcedEvent) GlobalEventRegistry.getInstance().parse(eventAsJsonObject, dittoHeaders); + } + /** * Check if any command is processed. */ @@ -757,4 +1011,97 @@ public String toString() { } + @Immutable + private final class EntityWithEvent { + @Nullable private final S entity; + private final long revision; + private final E event; + + private EntityWithEvent(@Nullable final S entity, final E event) { + this.entity = entity; + this.revision = event.getRevision(); + this.event = event; + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + + "entity=" + entity + + ", revision=" + revision + + ", event=" + event + + ']'; + } + } + + private final class HistoricalResultListener implements ResultVisitor { + + private final ActorRef sender; + private final DittoHeaders historicalDittoHeaders; + + private HistoricalResultListener(final ActorRef sender, final DittoHeaders historicalDittoHeaders) { + this.sender = sender; + this.historicalDittoHeaders = historicalDittoHeaders.toBuilder() + .removeHeader(DittoHeaderDefinition.RESPONSE_REQUIRED.getKey()) + .build(); + } + + @Override + public void onMutation(final Command command, final E event, final WithDittoHeaders response, + final boolean becomeCreated, final boolean becomeDeleted) { + throw new UnsupportedOperationException("Mutating historical entity not supported."); + } + + @Override + public void onStagedMutation(final Command command, final CompletionStage event, + final CompletionStage response, + final boolean becomeCreated, final boolean becomeDeleted) { + throw new UnsupportedOperationException("Mutating historical entity not supported."); + } + + @Override + public void onQuery(final Command command, final WithDittoHeaders response) { + if (command.getDittoHeaders().isResponseRequired()) { + final WithDittoHeaders theResponseToSend; + if (response instanceof DittoHeadersSettable dittoHeadersSettable) { + final DittoHeaders queryCommandHeaders = response.getDittoHeaders(); + final DittoHeaders adjustedHeaders = queryCommandHeaders.toBuilder() + .putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), + historicalDittoHeaders.toJson().toString()) + .build(); + theResponseToSend = dittoHeadersSettable.setDittoHeaders(adjustedHeaders); + } else { + theResponseToSend = response; + } + notifySender(sender, theResponseToSend); + } + } + + @Override + public void onStagedQuery(final Command command, final CompletionStage response) { + if (command.getDittoHeaders().isResponseRequired()) { + response.thenAccept(r -> { + final WithDittoHeaders theResponseToSend; + if (response instanceof DittoHeadersSettable dittoHeadersSettable) { + final DittoHeaders queryCommandHeaders = r.getDittoHeaders(); + final DittoHeaders adjustedHeaders = queryCommandHeaders.toBuilder() + .putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), + historicalDittoHeaders.toJson().toString()) + .build(); + theResponseToSend = dittoHeadersSettable.setDittoHeaders(adjustedHeaders); + } else { + theResponseToSend = r; + } + notifySender(sender, theResponseToSend); + }); + } + } + + @Override + public void onError(final DittoRuntimeException error, + final Command errorCausingCommand) { + if (shouldSendResponse(errorCausingCommand.getDittoHeaders())) { + notifySender(sender, error); + } + } + } } diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceSupervisor.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceSupervisor.java index 906df54340a..d1493354a53 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceSupervisor.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceSupervisor.java @@ -14,6 +14,8 @@ import java.text.MessageFormat; import java.time.Duration; +import java.time.Instant; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ThreadLocalRandom; @@ -22,21 +24,28 @@ import javax.annotation.Nullable; +import org.bson.BsonDocument; import org.eclipse.ditto.base.api.commands.sudo.SudoCommand; import org.eclipse.ditto.base.model.entity.id.EntityId; import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.DittoHeadersBuilder; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.WithResource; import org.eclipse.ditto.base.model.signals.WithType; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry; import org.eclipse.ditto.base.service.actors.ShutdownBehaviour; import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOff; import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOffConfig; +import org.eclipse.ditto.base.service.config.supervision.LocalAskTimeoutConfig; import org.eclipse.ditto.base.service.signaltransformer.SignalTransformer; import org.eclipse.ditto.base.service.signaltransformer.SignalTransformers; import org.eclipse.ditto.internal.utils.akka.actors.AbstractActorWithStashWithTimers; @@ -49,11 +58,16 @@ import org.eclipse.ditto.internal.utils.metrics.instruments.timer.PreparedTimer; import org.eclipse.ditto.internal.utils.metrics.instruments.timer.StartedTimer; import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces; +import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; +import org.eclipse.ditto.internal.utils.persistence.mongo.DittoBsonJson; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.tracing.DittoTracing; import org.eclipse.ditto.internal.utils.tracing.span.SpanOperationName; +import org.eclipse.ditto.json.JsonObject; import com.typesafe.config.Config; +import akka.NotUsed; import akka.actor.ActorRef; import akka.actor.ActorSystem; import akka.actor.OneForOneStrategy; @@ -69,6 +83,9 @@ import akka.japi.pf.ReceiveBuilder; import akka.pattern.AskTimeoutException; import akka.pattern.Patterns; +import akka.persistence.query.EventEnvelope; +import akka.stream.javadsl.Source; +import akka.stream.javadsl.StreamRefs; /** * Sharded Supervisor of persistent actors. It: @@ -84,12 +101,6 @@ public abstract class AbstractPersistenceSupervisor> extends AbstractActorWithStashWithTimers { - /** - * Timeout for local actor invocations - a small timeout should be more than sufficient as those are just method - * calls. - */ - protected static final Duration DEFAULT_LOCAL_ASK_TIMEOUT = Duration.ofSeconds(5); - private static final String ENFORCEMENT_TIMER = "enforcement"; private static final String ENFORCEMENT_TIMER_SEGMENT_ENFORCEMENT = "enf"; private static final String ENFORCEMENT_TIMER_SEGMENT_PROCESSING = "process"; @@ -110,13 +121,15 @@ public abstract class AbstractPersistenceSupervisor { log.error(error, "Got error in child. Stopping child actor for entityID <{}>.", entityId); @@ -185,6 +199,12 @@ protected AbstractPersistenceSupervisor(@Nullable final ActorRef persistenceActo */ protected abstract ExponentialBackOffConfig getExponentialBackOffConfig(); + /** + * Read from actor context. Called in constructor. + * @return local ASK timeout config; + */ + protected abstract LocalAskTimeoutConfig getLocalAskTimeoutConfig(); + /** * Get the shutdown behavior appropriate for this actor. * @@ -217,10 +237,84 @@ protected Receive activeBehaviour(final Runnable matchProcessNextTwinMessageBeha .match(SudoCommand.class, this::forwardSudoCommandToChildIfAvailable) .match(WithDittoHeaders.class, w -> w.getDittoHeaders().isSudo(), this::forwardDittoSudoToChildIfAvailable) + .match(SubscribeForPersistedEvents.class, this::handleStreamPersistedEvents) .matchAny(matchAnyBehavior) .build(); } + private void handleStreamPersistedEvents(final SubscribeForPersistedEvents subscribeForPersistedEvents) { + + final EntityId commandEntityId = subscribeForPersistedEvents.getEntityId(); + final String persistenceId = commandEntityId.getEntityType() + ":" + commandEntityId; + log.info("Starting to stream persisted events for pid <{}>: {}", persistenceId, subscribeForPersistedEvents); + + final Optional fromHistoricalTimestamp = subscribeForPersistedEvents.getFromHistoricalTimestamp(); + final Optional toHistoricalTimestamp = subscribeForPersistedEvents.getToHistoricalTimestamp(); + final Source startRevisionSource = fromHistoricalTimestamp + .map(fromTs -> mongoReadJournal.getLastSnapshotSequenceNumberBeforeTimestamp(persistenceId, fromTs) + .mergePrioritized( + Source.single(subscribeForPersistedEvents.getFromHistoricalRevision()), + 2, + 1, + false + ) + ) + .orElseGet(() -> Source.single(subscribeForPersistedEvents.getFromHistoricalRevision())); + + final ActorRef sender = getSender(); + askEnforcerChild(subscribeForPersistedEvents) + .whenComplete((enforcedStreamPersistedEvents, throwable) -> { + if (enforcedStreamPersistedEvents instanceof DittoRuntimeException dre) { + log.withCorrelationId(subscribeForPersistedEvents) + .info("Got DittoRuntimeException handling SubscribeForPersistedEvents: " + + "<{}: {}>", dre.getClass().getSimpleName(), dre.getMessage()); + sender.tell(dre, getSelf()); + } else if (null != enforcedStreamPersistedEvents) { + final var sourceRef = startRevisionSource + .flatMapConcat(startRevision -> mongoReadJournal.currentEventsByPersistenceId( + persistenceId, + startRevision, + subscribeForPersistedEvents.getToHistoricalRevision() + )) + .map(eventEnvelope -> + mapJournalEntryToEvent( + (SubscribeForPersistedEvents) enforcedStreamPersistedEvents, eventEnvelope)) + .filter(event -> + fromHistoricalTimestamp.flatMap(instant -> + event.getTimestamp().map(eventTs -> eventTs.isAfter(instant)) + ).orElse(true) + ) + .takeWhile(event -> + toHistoricalTimestamp.flatMap(instant -> + event.getTimestamp().map(eventTs -> eventTs.isBefore(instant)) + ).orElse(true) + ) + .runWith(StreamRefs.sourceRef(), getContext().getSystem()); + sender.tell(sourceRef, getSelf()); + } else if (null != throwable) { + log.withCorrelationId(subscribeForPersistedEvents) + .warning(throwable, "Got throwable: <{}: {}>", throwable.getClass().getSimpleName(), + throwable.getMessage()); + } + }); + } + + private Event mapJournalEntryToEvent(final SubscribeForPersistedEvents enforcedSubscribeForPersistedEvents, + final EventEnvelope eventEnvelope) { + + final BsonDocument event = (BsonDocument) eventEnvelope.event(); + final JsonObject eventAsJsonObject = DittoBsonJson.getInstance() + .serialize(event); + + final DittoHeadersBuilder dittoHeadersBuilder = enforcedSubscribeForPersistedEvents.getDittoHeaders() + .toBuilder(); + eventAsJsonObject.getValue(AbstractMongoEventAdapter.HISTORICAL_EVENT_HEADERS) + .ifPresent(obj -> dittoHeadersBuilder.putHeader( + DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), obj.toString()) + ); + return GlobalEventRegistry.getInstance().parse(eventAsJsonObject, dittoHeadersBuilder.build()); + } + /** * Create a builder for an exception to report unavailability of the entity. * @@ -252,6 +346,19 @@ protected CompletionStage modifyTargetActorCommandResponse(final Signal< return CompletableFuture.completedStage(persistenceCommandResponse); } + + /** + * Hook for handling unexpected PersistenceActor exceptions before response is sent back to the SupervisorActor. + * + * @param enforcedCommand the enforced initial command + * @param throwable the throwable + * @return a new {@link java.util.concurrent.CompletionStage} failed with the initial throwable. + */ + protected CompletionStage handleTargetActorAndEnforcerException(final Signal enforcedCommand, + final Throwable throwable) { + return CompletableFuture.failedFuture(throwable); + } + /** * Return a preferably static supervisor strategy for this actor. By default, child actor is stopped when killed * or failing, triggering restart after exponential back-off. @@ -319,7 +426,7 @@ protected void becomeCorrupted() { * failed due to lacking permissions. */ protected CompletionStage askEnforcerChild(final Signal signal) { - return Patterns.ask(enforcerChild, signal, DEFAULT_LOCAL_ASK_TIMEOUT); + return Patterns.ask(enforcerChild, signal, localAskTimeout); } /** @@ -454,7 +561,7 @@ protected CompletionStage getTargetActorForSendingEn new TargetActorWithMessage( persistenceActorChild, message, - shouldSendResponse ? defaultLocalAskTimeout : Duration.ZERO, + shouldSendResponse ? localAskTimeout : Duration.ZERO, Function.identity() )); } else { @@ -676,6 +783,7 @@ private void enforceAndForwardToTargetActor(final Object message) { final var syncCs = signalTransformer.apply(signal) .whenComplete((result, error) -> handleOptionalTransformationException(signal, error, sender)) .thenCompose(transformed -> enforceSignalAndForwardToTargetActor((S) transformed, sender) + .exceptionallyCompose(error -> handleTargetActorAndEnforcerException(transformed, error)) .whenComplete((response, throwable) -> handleSignalEnforcementResponse(response, throwable, transformed, sender) )) @@ -721,8 +829,9 @@ private void handleSignalEnforcementResponse(@Nullable final Object response, "forwarding to target actor, telling sender: {}", dre); sender.tell(dre, getSelf()); } else if (response instanceof Status.Success success) { - log.debug("Ignoring Status.Success message as expected 'to be ignored' outcome: <{}>", success); + log.withCorrelationId(signal).debug("Ignoring Status.Success message as expected 'to be ignored' outcome: <{}>", success); } else if (null != response) { + log.withCorrelationId(signal).debug("Sending response: <{}> back to sender: <{}>", response, sender.path()); sender.tell(response, getSelf()); } else { log.withCorrelationId(signal) diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/EmptyEvent.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/EmptyEvent.java index 4d4d6c1df13..51d6a0430e0 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/EmptyEvent.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/EmptyEvent.java @@ -56,7 +56,7 @@ public final class EmptyEvent implements Event { static final String NAME = "empty-event"; - static final String TYPE = TYPE_PREFIX + NAME; + public static final String TYPE = TYPE_PREFIX + NAME; private static final JsonFieldDefinition JSON_EFFECT = JsonFactory.newJsonValueFieldDefinition("effect", FieldType.REGULAR, JsonSchemaVersion.V_2); diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/Cleanup.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/Cleanup.java index 461447f692e..08ba16a5357 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/Cleanup.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/Cleanup.java @@ -16,6 +16,7 @@ import static org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal.S_ID; import static org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal.S_SN; +import java.time.Duration; import java.util.List; import java.util.function.Supplier; import java.util.stream.LongStream; @@ -35,6 +36,7 @@ final class Cleanup { private final MongoReadJournal readJournal; private final Materializer materializer; private final Supplier> responsibilitySupplier; + private final Duration historyRetentionDuration; private final int readBatchSize; private final int deleteBatchSize; private final boolean deleteFinalDeletedSnapshot; @@ -42,6 +44,7 @@ final class Cleanup { Cleanup(final MongoReadJournal readJournal, final Materializer materializer, final Supplier> responsibilitySupplier, + final Duration historyRetentionDuration, final int readBatchSize, final int deleteBatchSize, final boolean deleteFinalDeletedSnapshot) { @@ -49,6 +52,7 @@ final class Cleanup { this.readJournal = readJournal; this.materializer = materializer; this.responsibilitySupplier = responsibilitySupplier; + this.historyRetentionDuration = historyRetentionDuration; this.readBatchSize = readBatchSize; this.deleteBatchSize = deleteBatchSize; this.deleteFinalDeletedSnapshot = deleteFinalDeletedSnapshot; @@ -59,8 +63,12 @@ static Cleanup of(final CleanupConfig config, final Materializer materializer, final Supplier> responsibilitySupplier) { - return new Cleanup(readJournal, materializer, responsibilitySupplier, config.getReadsPerQuery(), - config.getWritesPerCredit(), config.shouldDeleteFinalDeletedSnapshot()); + return new Cleanup(readJournal, materializer, responsibilitySupplier, + config.getHistoryRetentionDuration(), + config.getReadsPerQuery(), + config.getWritesPerCredit(), + config.shouldDeleteFinalDeletedSnapshot() + ); } Source, NotUsed> getCleanupStream(final String lowerBound) { @@ -68,7 +76,7 @@ Source, NotUsed> getCleanupStream(final String lo } private Source getSnapshotRevisions(final String lowerBound) { - return readJournal.getNewestSnapshotsAbove(lowerBound, readBatchSize, true, materializer) + return readJournal.getNewestSnapshotsAbove(lowerBound, readBatchSize, true, historyRetentionDuration, materializer) .map(document -> new SnapshotRevision(document.getString(S_ID), document.getLong(S_SN), "DELETED".equals(document.getString(LIFECYCLE)))) @@ -92,7 +100,8 @@ private Source, NotUsed> cleanUpEvents(final Snap } else { final List upperBounds = getSnUpperBoundsPerBatch(minSnOpt.orElseThrow(), sr.sn); return Source.from(upperBounds).map(upperBound -> Source.lazySource(() -> - readJournal.deleteEvents(sr.pid, upperBound - deleteBatchSize + 1, upperBound) + readJournal + .deleteEvents(sr.pid, upperBound - deleteBatchSize + 1, upperBound) .map(result -> new CleanupResult(CleanupResult.Type.EVENTS, sr, result)) ).mapMaterializedValue(ignored -> NotUsed.getInstance())); } diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupConfig.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupConfig.java index ece8620f307..a84469ab36c 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupConfig.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupConfig.java @@ -41,7 +41,7 @@ static CleanupConfig of(final Config config) { * @param config the config values to set. * @return the new cleanup config object. */ - CleanupConfig setAll(final Config config); + CleanupConfig setAll(Config config); /** * Return whether background cleanup is enabled. @@ -50,6 +50,16 @@ static CleanupConfig of(final Config config) { */ boolean isEnabled(); + /** + * Returns the duration of how long to "keep" events and snapshots before being allowed to remove them in scope + * of cleanup. + * If this e.g. is set to 30 days - then effectively an event history of 30 days would be available via the read + * journal. + * + * @return the history retention duration. + */ + Duration getHistoryRetentionDuration(); + /** * Returns quiet period between cleanup streams. * @@ -118,6 +128,12 @@ enum ConfigValue implements KnownConfigValue { */ ENABLED("enabled", true), + /** + * History retention duration. + * Events and snapshots are kept at least that long before cleaning them up from the persistence. + */ + HISTORY_RETENTION_DURATION("history-retention-duration", Duration.ofDays(0L)), + /** * Quiet period. */ diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/DefaultCleanupConfig.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/DefaultCleanupConfig.java index 79b03f78e0a..b2146a62abe 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/DefaultCleanupConfig.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/DefaultCleanupConfig.java @@ -29,6 +29,7 @@ final class DefaultCleanupConfig implements CleanupConfig { static final String CONFIG_PATH = "cleanup"; private final boolean enabled; + private final Duration historyRetentionDuration; private final Duration quietPeriod; private final Duration interval; private final Duration timerThreshold; @@ -38,6 +39,7 @@ final class DefaultCleanupConfig implements CleanupConfig { private final boolean deleteFinalDeletedSnapshot; DefaultCleanupConfig(final boolean enabled, + final Duration historyRetentionDuration, final Duration quietPeriod, final Duration interval, final Duration timerThreshold, @@ -46,6 +48,7 @@ final class DefaultCleanupConfig implements CleanupConfig { final int writesPerCredit, final boolean deleteFinalDeletedSnapshot) { this.enabled = enabled; + this.historyRetentionDuration = historyRetentionDuration; this.quietPeriod = quietPeriod; this.interval = interval; this.timerThreshold = timerThreshold; @@ -57,6 +60,7 @@ final class DefaultCleanupConfig implements CleanupConfig { DefaultCleanupConfig(final ScopedConfig conf) { this.enabled = conf.getBoolean(ConfigValue.ENABLED.getConfigPath()); + this.historyRetentionDuration = conf.getNonNegativeDurationOrThrow(ConfigValue.HISTORY_RETENTION_DURATION); this.quietPeriod = conf.getNonNegativeAndNonZeroDurationOrThrow(ConfigValue.QUIET_PERIOD); this.interval = conf.getNonNegativeAndNonZeroDurationOrThrow(ConfigValue.INTERVAL); this.timerThreshold = conf.getNonNegativeAndNonZeroDurationOrThrow(ConfigValue.TIMER_THRESHOLD); @@ -70,6 +74,7 @@ final class DefaultCleanupConfig implements CleanupConfig { public Config render() { final Map configMap = Map.of( ConfigValue.ENABLED.getConfigPath(), enabled, + ConfigValue.HISTORY_RETENTION_DURATION.getConfigPath(), historyRetentionDuration, ConfigValue.QUIET_PERIOD.getConfigPath(), quietPeriod, ConfigValue.INTERVAL.getConfigPath(), interval, ConfigValue.TIMER_THRESHOLD.getConfigPath(), timerThreshold, @@ -91,6 +96,11 @@ public boolean isEnabled() { return enabled; } + @Override + public Duration getHistoryRetentionDuration() { + return historyRetentionDuration; + } + @Override public Duration getQuietPeriod() { return quietPeriod; @@ -128,9 +138,9 @@ public boolean shouldDeleteFinalDeletedSnapshot() { @Override public boolean equals(final Object o) { - if (o instanceof DefaultCleanupConfig) { - final DefaultCleanupConfig that = (DefaultCleanupConfig) o; + if (o instanceof DefaultCleanupConfig that) { return enabled == that.enabled && + Objects.equals(historyRetentionDuration, that.historyRetentionDuration) && Objects.equals(quietPeriod, that.quietPeriod) && Objects.equals(interval, that.interval) && Objects.equals(timerThreshold, that.timerThreshold) && @@ -145,21 +155,22 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(enabled, quietPeriod, interval, timerThreshold, creditsPerBatch, readsPerQuery, - writesPerCredit, deleteFinalDeletedSnapshot); + return Objects.hash(enabled, historyRetentionDuration, quietPeriod, interval, timerThreshold, creditsPerBatch, + readsPerQuery, writesPerCredit, deleteFinalDeletedSnapshot); } @Override public String toString() { - return getClass().getSimpleName() + - "[enabled=" + enabled + - ",quietPeriod=" + quietPeriod + - ",interval=" + interval + - ",timerThreshold=" + timerThreshold + - ",creditPerBatch=" + creditsPerBatch + - ",readsPerQuery=" + readsPerQuery + - ",writesPerCredit=" + writesPerCredit + - ",deleteFinalDeletedSnapshot=" + deleteFinalDeletedSnapshot + + return getClass().getSimpleName() + "[" + + "enabled=" + enabled + + ", minAgeFromNow=" + historyRetentionDuration + + ", quietPeriod=" + quietPeriod + + ", interval=" + interval + + ", timerThreshold=" + timerThreshold + + ", creditPerBatch=" + creditsPerBatch + + ", readsPerQuery=" + readsPerQuery + + ", writesPerCredit=" + writesPerCredit + + ", deleteFinalDeletedSnapshot=" + deleteFinalDeletedSnapshot + "]"; } diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/commands/AbstractCommandStrategies.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/commands/AbstractCommandStrategies.java index 563995a5589..e49461cecf3 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/commands/AbstractCommandStrategies.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/commands/AbstractCommandStrategies.java @@ -97,7 +97,7 @@ protected Result doApply(final Context context, @Nullable final S entity, if (commandStrategy != null) { context.getLog().withCorrelationId(command) .debug("Applying command <{}>", command); - return commandStrategy.apply(context, entity, nextRevision, command).map(x -> x); + return (Result) commandStrategy.apply(context, entity, nextRevision, command); } else { // this may happen when subclasses override the "isDefined" condition. return unhandled(context, entity, nextRevision, command); diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/etags/AbstractConditionHeaderCheckingCommandStrategy.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/etags/AbstractConditionHeaderCheckingCommandStrategy.java index 93092c65e2c..c38ade2a634 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/etags/AbstractConditionHeaderCheckingCommandStrategy.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/etags/AbstractConditionHeaderCheckingCommandStrategy.java @@ -82,9 +82,12 @@ public Result apply(final Context context, @Nullable final S entity, final context.getLog().withCorrelationId(command) .debug("Validating conditional headers with currentETagValue <{}> on command <{}>.", currentETagValue, command); + + final C adjustedCommand; try { getValidator().checkConditionalHeaders(command, currentETagValue); - context.getLog().withCorrelationId(command) + adjustedCommand = getValidator().applyIfEqualHeader(command, entity); + context.getLog().withCorrelationId(adjustedCommand) .debug("Validating conditional headers succeeded."); } catch (final DittoRuntimeException dre) { context.getLog().withCorrelationId(command) @@ -92,7 +95,7 @@ public Result apply(final Context context, @Nullable final S entity, final return ResultFactory.newErrorResult(dre, command); } - return super.apply(context, entity, nextRevision, command); + return super.apply(context, entity, nextRevision, adjustedCommand); } @Override diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/AbstractEventStrategies.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/AbstractEventStrategies.java index a9d9439a40d..db2a147718f 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/AbstractEventStrategies.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/AbstractEventStrategies.java @@ -20,7 +20,7 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; -import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +31,7 @@ * @param the type of the entity */ @Immutable -public abstract class AbstractEventStrategies, S> implements EventStrategy { +public abstract class AbstractEventStrategies, S> implements EventStrategy { protected final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/EventStrategy.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/EventStrategy.java index d9a2f7735ac..cebb7a29b10 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/EventStrategy.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/EventStrategy.java @@ -14,7 +14,7 @@ import javax.annotation.Nullable; -import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; /** * This interface represents a strategy for handling events in a persistent actor. @@ -23,7 +23,7 @@ * @param the type of the entity */ @FunctionalInterface -public interface EventStrategy, S> { +public interface EventStrategy, S> { /** * Applies an event to an entity. diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/EmptyResult.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/EmptyResult.java index ba93b8f9047..aff9969d19b 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/EmptyResult.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/EmptyResult.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.internal.utils.persistentactors.results; +import java.util.concurrent.CompletionStage; import java.util.function.Function; import org.eclipse.ditto.base.model.signals.events.Event; @@ -46,6 +47,11 @@ public > Result map(final Function mappingFunction) return getInstance(); } + @Override + public > Result mapStages(final Function, CompletionStage> mappingFunction) { + return getInstance(); + } + @Override public String toString() { return this.getClass().getSimpleName() + " []"; diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ErrorResult.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ErrorResult.java index 307a960fb5f..4cf6822691e 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ErrorResult.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ErrorResult.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.internal.utils.persistentactors.results; +import java.util.concurrent.CompletionStage; import java.util.function.Function; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; @@ -48,4 +49,9 @@ public void accept(final ResultVisitor visitor) { public > Result map(final Function mappingFunction) { return new ErrorResult<>(dittoRuntimeException, errorCausingCommand); } + + @Override + public > Result mapStages(final Function, CompletionStage> mappingFunction) { + return new ErrorResult<>(dittoRuntimeException, errorCausingCommand); + } } diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/MutationResult.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/MutationResult.java index 70458b19cd0..afdbd71156b 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/MutationResult.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/MutationResult.java @@ -12,8 +12,11 @@ */ package org.eclipse.ditto.internal.utils.persistentactors.results; +import java.util.concurrent.CompletionStage; import java.util.function.Function; +import javax.annotation.Nullable; + import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.events.Event; @@ -26,28 +29,46 @@ public final class MutationResult> implements Result { private final Command command; - private final E eventToPersist; - private final WithDittoHeaders response; + @Nullable private final E eventToPersist; + @Nullable private final WithDittoHeaders response; + @Nullable private final CompletionStage eventToPersistStage; + @Nullable private final CompletionStage responseStage; private final boolean becomeCreated; private final boolean becomeDeleted; - MutationResult(final Command command, final E eventToPersist, final WithDittoHeaders response, + MutationResult(final Command command, + @Nullable final E eventToPersist, + @Nullable final WithDittoHeaders response, + @Nullable final CompletionStage eventToPersistStage, + @Nullable final CompletionStage responseStage, final boolean becomeCreated, final boolean becomeDeleted) { this.command = command; this.eventToPersist = eventToPersist; this.response = response; + this.eventToPersistStage = eventToPersistStage; + this.responseStage = responseStage; this.becomeCreated = becomeCreated; this.becomeDeleted = becomeDeleted; } @Override public void accept(final ResultVisitor visitor) { - visitor.onMutation(command, eventToPersist, response, becomeCreated, becomeDeleted); + if (eventToPersistStage != null && responseStage != null) { + visitor.onStagedMutation(command, eventToPersistStage, responseStage, becomeCreated, becomeDeleted); + } else { + visitor.onMutation(command, eventToPersist, response, becomeCreated, becomeDeleted); + } } @Override public > Result map(final Function mappingFunction) { - return new MutationResult<>(command, mappingFunction.apply(eventToPersist), response, becomeCreated, + return new MutationResult<>(command, mappingFunction.apply(eventToPersist), response, null, responseStage, becomeCreated, + becomeDeleted); + } + + @Override + public > Result mapStages(final Function, CompletionStage> mappingFunction) { + return new MutationResult<>(command, null, response, mappingFunction.apply(eventToPersistStage), responseStage, becomeCreated, becomeDeleted); } @@ -57,6 +78,8 @@ public String toString() { "command=" + command + ", eventToPersist=" + eventToPersist + ", response=" + response + + ", eventToPersistStage=" + eventToPersistStage + + ", responseStage=" + responseStage + ", becomeCreated=" + becomeCreated + ", becomeDeleted=" + becomeDeleted + ']'; diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/QueryResult.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/QueryResult.java index 1f6dfbae559..9a82284f885 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/QueryResult.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/QueryResult.java @@ -12,8 +12,11 @@ */ package org.eclipse.ditto.internal.utils.persistentactors.results; +import java.util.concurrent.CompletionStage; import java.util.function.Function; +import javax.annotation.Nullable; + import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.events.Event; @@ -26,11 +29,15 @@ public final class QueryResult> implements Result { private final Command command; - private final WithDittoHeaders response; + @Nullable private final WithDittoHeaders response; + @Nullable private final CompletionStage responseStage; - QueryResult(final Command command, final WithDittoHeaders response) { + QueryResult(final Command command, + @Nullable final WithDittoHeaders response, + @Nullable final CompletionStage responseStage) { this.command = command; this.response = response; + this.responseStage = responseStage; } @Override @@ -38,16 +45,26 @@ public String toString() { return this.getClass().getSimpleName() + " [" + "command=" + command + ", response=" + response + + ", responseStage=" + responseStage + ']'; } @Override public void accept(final ResultVisitor visitor) { - visitor.onQuery(command, response); + if (responseStage != null) { + visitor.onStagedQuery(command, responseStage); + } else { + visitor.onQuery(command, response); + } } @Override public > Result map(final Function mappingFunction) { - return new QueryResult<>(command, response); + return new QueryResult<>(command, response, null); + } + + @Override + public > Result mapStages(final Function, CompletionStage> mappingFunction) { + return new QueryResult<>(command, null, responseStage); } } diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/Result.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/Result.java index 7cd94f00937..cb92d2b9071 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/Result.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/Result.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.internal.utils.persistentactors.results; +import java.util.concurrent.CompletionStage; import java.util.function.Function; import org.eclipse.ditto.base.model.signals.events.Event; @@ -26,7 +27,7 @@ public interface Result> { * * @param visitor the visitor to evaluate the result, typically the persistent actor itself. */ - void accept(final ResultVisitor visitor); + void accept(ResultVisitor visitor); /** * Convert the result with a function. @@ -34,10 +35,18 @@ public interface Result> { * @param mappingFunction the mapping function. * @param the new event type of the result. * @return the new result. - * @since 2.0.0 */ > Result map(Function mappingFunction); + /** + * Convert the result with a function. + * + * @param mappingFunction the mapping function. + * @param the new event type of the result. + * @return the new result. + */ + > Result mapStages(Function, CompletionStage> mappingFunction); + /** * @return the empty result */ diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ResultFactory.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ResultFactory.java index cbfa5f312ba..36bed2db20b 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ResultFactory.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ResultFactory.java @@ -12,6 +12,8 @@ */ package org.eclipse.ditto.internal.utils.persistentactors.results; +import java.util.concurrent.CompletionStage; + import javax.annotation.concurrent.Immutable; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; @@ -41,7 +43,25 @@ private ResultFactory() { public static > Result newMutationResult(final Command command, final E eventToPersist, final WithDittoHeaders response) { - return new MutationResult<>(command, eventToPersist, response, false, false); + return new MutationResult<>(command, eventToPersist, response, null, null, + false, false); + } + + /** + * Create a mutation result. + * + * @param command command that caused the mutation. + * @param eventToPersist event of the mutation. + * @param response response of the command. + * @param type of the event. + * @return the result. + */ + public static > Result newMutationResult(final Command command, + final CompletionStage eventToPersist, + final CompletionStage response) { + + return new MutationResult<>(command, null, null, eventToPersist, response, + false, false); } /** @@ -61,7 +81,29 @@ public static > Result newMutationResult(final Command final boolean becomeCreated, final boolean becomeDeleted) { - return new MutationResult<>(command, eventToPersist, response, becomeCreated, becomeDeleted); + return new MutationResult<>(command, eventToPersist, response, null, null, + becomeCreated, becomeDeleted); + } + + /** + * Create a mutation result. + * + * @param command command that caused the mutation. + * @param eventToPersist event of the mutation. + * @param response response of the command. + * @param becomeCreated whether the actor should behave as if the entity is created. + * @param becomeDeleted whether the actor should behave as if the entity is deleted. + * @param type of the event. + * @return the result. + */ + public static > Result newMutationResult(final Command command, + final CompletionStage eventToPersist, + final CompletionStage response, + final boolean becomeCreated, + final boolean becomeDeleted) { + + return new MutationResult<>(command, null, null, eventToPersist, response, + becomeCreated, becomeDeleted); } /** @@ -87,7 +129,20 @@ public static > Result newErrorResult(final DittoRuntimeEx */ public static > Result newQueryResult(final Command command, final WithDittoHeaders response) { - return new QueryResult<>(command, response); + return new QueryResult<>(command, response, null); + } + + /** + * Create a query result. + * + * @param command the query command. + * @param response the response. + * @param type of events (irrelevant). + * @return the result. + */ + public static > Result newQueryResult(final Command command, + final CompletionStage response) { + return new QueryResult<>(command, null, response); } /** diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ResultVisitor.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ResultVisitor.java index e2b8cebe8cd..cd3322d5f86 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ResultVisitor.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/results/ResultVisitor.java @@ -12,6 +12,8 @@ */ package org.eclipse.ditto.internal.utils.persistentactors.results; +import java.util.concurrent.CompletionStage; + import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.signals.commands.Command; @@ -40,8 +42,19 @@ default void onEmpty() { * @param becomeCreated whether the actor should behave as if the entity is created. * @param becomeDeleted whether the actor should behave as if the entity is deleted. */ - void onMutation(Command command, E event, WithDittoHeaders response, boolean becomeCreated, - boolean becomeDeleted); + void onMutation(Command command, E event, WithDittoHeaders response, boolean becomeCreated, boolean becomeDeleted); + + /** + * Evaluate a mutation result. + * + * @param command command that caused the mutation. + * @param event event of the mutation. + * @param response response of the command. + * @param becomeCreated whether the actor should behave as if the entity is created. + * @param becomeDeleted whether the actor should behave as if the entity is deleted. + */ + void onStagedMutation(Command command, CompletionStage event, CompletionStage response, + boolean becomeCreated, boolean becomeDeleted); /** * Evaluate a query result. @@ -51,6 +64,14 @@ void onMutation(Command command, E event, WithDittoHeaders response, boolean */ void onQuery(Command command, WithDittoHeaders response); + /** + * Evaluate a query result. + * + * @param command the query command. + * @param response the response. + */ + void onStagedQuery(Command command, CompletionStage response); + /** * Evaluate an error result. * diff --git a/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupTest.java b/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupTest.java index c9ac8417f59..3f74f0ac52a 100644 --- a/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupTest.java +++ b/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -67,7 +68,8 @@ public void emptyStream() { when(mongoReadJournal.getNewestSnapshotsAbove(any(), anyInt(), eq(true), any(), any())) .thenReturn(Source.empty()); - final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), 1, 1, true); + final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), + Duration.ZERO, 1, 1, true); final var result = underTest.getCleanupStream("") .flatMapConcat(x -> x) .runWith(Sink.seq(), materializer) @@ -94,7 +96,8 @@ public void deleteFinalDeletedSnapshot() { invocation.getArgument(1) * 1000L + invocation.getArgument(2) * 10L))) .when(mongoReadJournal).deleteSnapshots(any(), anyLong(), anyLong()); - final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), 1, 4, true); + final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), + Duration.ZERO, 1, 4, true); final var result = underTest.getCleanupStream("") .flatMapConcat(x -> x) @@ -127,7 +130,8 @@ public void excludeFinalDeletedSnapshot() { invocation.getArgument(1) * 1000L + invocation.getArgument(2) * 10L))) .when(mongoReadJournal).deleteSnapshots(any(), anyLong(), anyLong()); - final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), 1, 4, false); + final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), + Duration.ZERO, 1, 4, false); final var result = underTest.getCleanupStream("") .flatMapConcat(x -> x) @@ -168,7 +172,8 @@ public void ignorePidsNotResponsibleFor() { .when(mongoReadJournal).deleteSnapshots(any(), anyLong(), anyLong()); // WHEN: the instance is responsible for 1/3 of the 3 PIDs - final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(2, 3), 1, 4, false); + final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(2, 3), + Duration.ZERO, 1, 4, false); final var result = underTest.getCleanupStream("") .flatMapConcat(x -> x) diff --git a/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CreditsTest.java b/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CreditsTest.java index e72be46a6db..e64b21507bf 100644 --- a/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CreditsTest.java +++ b/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CreditsTest.java @@ -134,7 +134,8 @@ public void onePersistenceWriteAllowedPerCredit() { // mock timer permits 1 batch of credit, after which no credit is given out final var mockTimerResult = new AtomicLong(0L); doAnswer(inv -> mockTimerResult.getAndSet(1001L)).when(mockTimer).getThenReset(); - final var cleanup = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), 1, 4, true); + final var cleanup = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), + Duration.ZERO, 1, 4, true); final var underTest = new Credits(getFastCreditConfig(4), mockTimer); final var log = Logging.getLogger(actorSystem, this); @@ -164,7 +165,7 @@ private Pair, TestSubscriber.Probe> material } private static CleanupConfig getFastCreditConfig(final int creditPerBatch) { - return new DefaultCleanupConfig(true, Duration.ZERO, Duration.ofMillis(100), Duration.ofNanos(1000), + return new DefaultCleanupConfig(true, Duration.ZERO, Duration.ZERO, Duration.ofMillis(100), Duration.ofNanos(1000), creditPerBatch, 100, 100, false); } } diff --git a/internal/utils/pubsub-things/src/main/java/org/eclipse/ditto/internal/utils/pubsubthings/DittoProtocolSubImpl.java b/internal/utils/pubsub-things/src/main/java/org/eclipse/ditto/internal/utils/pubsubthings/DittoProtocolSubImpl.java index 80fde2bfee7..661ec64a1db 100644 --- a/internal/utils/pubsub-things/src/main/java/org/eclipse/ditto/internal/utils/pubsubthings/DittoProtocolSubImpl.java +++ b/internal/utils/pubsub-things/src/main/java/org/eclipse/ditto/internal/utils/pubsubthings/DittoProtocolSubImpl.java @@ -25,6 +25,9 @@ import javax.annotation.Nullable; import org.eclipse.ditto.base.model.acks.AcknowledgementLabel; +import org.eclipse.ditto.base.model.acks.AcknowledgementLabelInvalidException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.internal.utils.pubsub.DistributedAcks; import org.eclipse.ditto.internal.utils.pubsub.DistributedSub; import org.eclipse.ditto.internal.utils.pubsub.StreamingType; @@ -131,9 +134,11 @@ public CompletionStage declareAcknowledgementLabels( return CompletableFuture.completedFuture(null); } - // don't complete the future with the exception this method emits as this is a bug in Ditto which we must escalate - // via the actor supervision strategy - ensureAcknowledgementLabelsAreFullyResolved(acknowledgementLabels); + try { + ensureAcknowledgementLabelsAreFullyResolved(acknowledgementLabels); + } catch (final DittoRuntimeException dre) { + return CompletableFuture.failedStage(dre); + } return distributedAcks.declareAcknowledgementLabels(acknowledgementLabels, subscriber, group) .thenApply(ack -> null); @@ -144,9 +149,10 @@ private static void ensureAcknowledgementLabelsAreFullyResolved(final Collection .filter(Predicate.not(AcknowledgementLabel::isFullyResolved)) .findFirst() .ifPresent(ackLabel -> { - // if this happens, this is a bug in the Ditto codebase! at this point the AckLabel must be resolved - throw new IllegalArgumentException("AcknowledgementLabel was not fully resolved while " + - "trying to declare it: " + ackLabel); + throw AcknowledgementLabelInvalidException.of(ackLabel, + "AcknowledgementLabel was not fully resolved while trying to declare it", + null, + DittoHeaders.empty()); }); } diff --git a/json/src/main/java/org/eclipse/ditto/json/JsonFactory.java b/json/src/main/java/org/eclipse/ditto/json/JsonFactory.java index d3b3d5527ed..08cc64bf9c3 100755 --- a/json/src/main/java/org/eclipse/ditto/json/JsonFactory.java +++ b/json/src/main/java/org/eclipse/ditto/json/JsonFactory.java @@ -27,6 +27,8 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; @@ -614,6 +616,45 @@ public static JsonFieldSelector newFieldSelector(@Nullable final String fieldSel return result; } + /** + * Returns a new JSON field selector by parsing the given string. If the JSON field selector string is {@code null} + * or empty this means that no fields were selected thus this method returns an empty JSON field selector. + *

    + * For example, the field selector string + *

    + * 
    +     * "thingId,attributes(acceleration,someData(foo,bar/baz)),features/key"
+     *
    + * would lead to a JSON field selector which consists of the following JSON pointers: + *
      + *
    • {@code "thingId"},
      • + *
    • {@code "attributes/acceleration"},
      • + *
    • {@code "attributes/someData/foo"},
      • + *
    • {@code "attributes/someData/bar/baz"},
      • + *
    • {@code "features/key"}.
      • + *
    + * + * @param fieldSelectorStrings strings to be transformed into a JSON field selector object. + * @param options the JsonParseOptions to apply when parsing the {@code fieldSelectorString}. + * @return a new JSON field selector. + * @throws JsonFieldSelectorInvalidException if {@code fieldSelectorString} is empty or if + * {@code fieldSelectorString} does not contain a closing parenthesis ({@code )}) for each opening parenthesis + * ({@code (}). + * @throws IllegalStateException if {@code fieldSelectorStrings} cannot be decoded as UTF-8. + * @since 3.3.0 + */ + public static JsonFieldSelector newFieldSelector(final Iterable fieldSelectorStrings, + final JsonParseOptions options) { + + final Collection selectors = StreamSupport.stream(fieldSelectorStrings.spliterator(), false) + .filter(fieldSelectorString -> !fieldSelectorString.isEmpty()) + .map(fieldSelectorString -> ImmutableJsonFieldSelectorFactory.newInstance(fieldSelectorString, options)) + .map(ImmutableJsonFieldSelectorFactory::newJsonFieldSelector) + .flatMap(foo -> foo.getPointers().stream()) + .collect(Collectors.toList()); + return newFieldSelector(selectors); + } + /** * Returns a new JSON field selector which is based on the given set of {@link JsonPointer}s. * If the set of JSON pointers string is empty this means that no fields were selected thus this method returns an diff --git a/json/src/main/java/org/eclipse/ditto/json/JsonMergePatch.java b/json/src/main/java/org/eclipse/ditto/json/JsonMergePatch.java index d1dee2ac365..415ecfac596 100644 --- a/json/src/main/java/org/eclipse/ditto/json/JsonMergePatch.java +++ b/json/src/main/java/org/eclipse/ditto/json/JsonMergePatch.java @@ -12,9 +12,11 @@ */ package org.eclipse.ditto.json; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -165,9 +167,11 @@ private static JsonObject mergeJsonObjects(final JsonObject jsonObject1, final J } }); + final List toBeNulledKeysByRegex = determineToBeNulledKeysByRegex(jsonObject1, jsonObject2); + // add fields of jsonObject2 not present in jsonObject1 jsonObject2.forEach(jsonField -> { - if (!jsonObject1.contains(jsonField.getKey())) { + if (!jsonObject1.contains(jsonField.getKey()) && !toBeNulledKeysByRegex.contains(jsonField.getKey())) { builder.set(jsonField); } }); @@ -175,6 +179,31 @@ private static JsonObject mergeJsonObjects(final JsonObject jsonObject1, final J return builder.build(); } + private static List determineToBeNulledKeysByRegex( + final JsonObject jsonObject1, + final JsonObject jsonObject2) { + + final List toBeNulledKeysByRegex = new ArrayList<>(); + final List keyRegexes = jsonObject1.getKeys().stream() + .filter(key -> key.toString().startsWith("{{") && key.toString().endsWith("}}")) + .collect(Collectors.toList()); + keyRegexes.forEach(keyRegex -> { + final String keyRegexWithoutCurly = keyRegex.toString().substring(2, keyRegex.length() - 2).trim(); + if (keyRegexWithoutCurly.startsWith("/") && keyRegexWithoutCurly.endsWith("/")) { + final String regexStr = keyRegexWithoutCurly.substring(1, keyRegexWithoutCurly.length() - 1); + final Pattern pattern = Pattern.compile(regexStr); + jsonObject1.getValue(keyRegex) + .filter(JsonValue::isNull) // only support deletion via regex, so only support "null" values + .ifPresent(keyRegexValue -> + jsonObject2.getKeys().stream() + .filter(key -> pattern.matcher(key).matches()) + .forEach(toBeNulledKeysByRegex::add) + ); + } + }); + return toBeNulledKeysByRegex; + } + /** * Applies this merge patch on the given json value. * diff --git a/json/src/test/java/org/eclipse/ditto/json/JsonFactoryTest.java b/json/src/test/java/org/eclipse/ditto/json/JsonFactoryTest.java index 73775e11a1f..23624da8e8a 100755 --- a/json/src/test/java/org/eclipse/ditto/json/JsonFactoryTest.java +++ b/json/src/test/java/org/eclipse/ditto/json/JsonFactoryTest.java @@ -381,7 +381,15 @@ public void newFieldSelectorFromPointerStringsReturnsExpected() { @Test public void newFieldSelectorFromNullStringIsEmpty() { - final JsonFieldSelector fieldSelector = JsonFactory.newFieldSelector(null, + final JsonFieldSelector fieldSelector = JsonFactory.newFieldSelector((String) null, + JsonFactory.newParseOptionsBuilder().withoutUrlDecoding().build()); + + assertThat(fieldSelector).isEmpty(); + } + + @Test + public void newFieldSelectorFromEmptyStringListIsEmpty() { + final JsonFieldSelector fieldSelector = JsonFactory.newFieldSelector(Collections.emptyList(), JsonFactory.newParseOptionsBuilder().withoutUrlDecoding().build()); assertThat(fieldSelector).isEmpty(); diff --git a/json/src/test/java/org/eclipse/ditto/json/JsonMergePatchTest.java b/json/src/test/java/org/eclipse/ditto/json/JsonMergePatchTest.java index 2e699e28ae6..f21da4bafa4 100644 --- a/json/src/test/java/org/eclipse/ditto/json/JsonMergePatchTest.java +++ b/json/src/test/java/org/eclipse/ditto/json/JsonMergePatchTest.java @@ -447,4 +447,81 @@ public void mergeFieldsFromBothObjectsRFC7396_TestCase13() { Assertions.assertThat(mergedObject).isEqualTo(expectedObject); } + @Test + public void removeFieldsUsingRegexWithNullValue() { + final JsonObject originalObject = JsonFactory.newObjectBuilder() + .set("a", JsonFactory.newObjectBuilder() + .set("2023-04-01", JsonValue.of(true)) + .set("2023-04-02", JsonValue.of("hello")) + .set("2023-04-03", JsonValue.of("darkness")) + .set("2023-04-04", JsonValue.of("my")) + .set("2023-04-05", JsonValue.of("old")) + .set("2023-04-06", JsonValue.of("friend")) + .build()) + .set("b", JsonFactory.newObjectBuilder() + .set("2023-04-01", JsonValue.of(true)) + .set("2023-04-02", JsonValue.of("hello")) + .set("2023-04-03", JsonValue.of("darkness")) + .set("2023-04-04", JsonValue.of("my")) + .set("2023-04-05", JsonValue.of("old")) + .set("2023-04-06", JsonValue.of("friend")) + .build()) + .set("c", JsonFactory.newObjectBuilder() + .set("some", JsonValue.of(true)) + .set("2023-04-02", JsonValue.of("hello")) + .set("totally-other", JsonValue.of("darkness")) + .set("foo", JsonValue.of("my")) + .build()) + .build(); + + final JsonObject objectToPatch = JsonFactory.newObjectBuilder() + .set("a", JsonFactory.newObjectBuilder() + .set("{{ /2023-04-.*/ }}", JsonValue.nullLiteral()) + .set("2023-05-01", JsonValue.of("new")) + .set("2023-05-02", JsonValue.of("catch")) + .set("2023-05-03", JsonValue.of("phrase")) + .build()) + .set("b", JsonFactory.newObjectBuilder() + .set("{{ /2023-04-01/ }}", JsonValue.nullLiteral()) + .set("{{ /^2023-04-03$/ }}", JsonValue.nullLiteral()) + .set("{{ /[0-9]{4}-04-.+4/ }}", JsonValue.nullLiteral()) + .set("2023-05-01", JsonValue.of("new")) + .set("2023-05-02", JsonValue.of("catch")) + .set("2023-05-03", JsonValue.of("phrase")) + .build()) + .set("c", JsonFactory.newObjectBuilder() + .set("{{ /.*/ }}", JsonValue.nullLiteral()) + .set("2023-05-01", JsonValue.of("new")) + .set("2023-05-02", JsonValue.of("catch")) + .set("2023-05-03", JsonValue.of("phrase")) + .build()) + .build(); + + final JsonValue expectedObject = JsonFactory.newObjectBuilder() + .set("a", JsonFactory.newObjectBuilder() + .set("2023-05-01", JsonValue.of("new")) + .set("2023-05-02", JsonValue.of("catch")) + .set("2023-05-03", JsonValue.of("phrase")) + .build()) + .set("b", JsonFactory.newObjectBuilder() + .set("2023-04-02", JsonValue.of("hello")) + .set("2023-04-05", JsonValue.of("old")) + .set("2023-04-06", JsonValue.of("friend")) + .set("2023-05-01", JsonValue.of("new")) + .set("2023-05-02", JsonValue.of("catch")) + .set("2023-05-03", JsonValue.of("phrase")) + .build()) + .set("c", JsonFactory.newObjectBuilder() + .set("2023-05-01", JsonValue.of("new")) + .set("2023-05-02", JsonValue.of("catch")) + .set("2023-05-03", JsonValue.of("phrase")) + .build()) + .build(); + + final JsonValue mergedObject = JsonMergePatch.of(objectToPatch).applyOn(originalObject); + + Assertions.assertThat(mergedObject).isEqualTo(expectedObject); + } + + } diff --git a/legal/3rd-party-dependencies/compile.txt b/legal/3rd-party-dependencies/compile.txt index af3c9eaa74b..fb51bfe11e1 100644 --- a/legal/3rd-party-dependencies/compile.txt +++ b/legal/3rd-party-dependencies/compile.txt @@ -1,15 +1,15 @@ ch.qos.logback:logback-classic:jar:1.2.11:compile ch.qos.logback:logback-core:jar:1.2.11:compile com.eclipsesource.minimal-json:minimal-json:jar:0.9.5:compile -com.fasterxml.jackson.core:jackson-annotations:jar:2.13.4:compile -com.fasterxml.jackson.core:jackson-core:jar:2.13.4:compile -com.fasterxml.jackson.core:jackson-databind:jar:2.13.4.2:compile -com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar:2.13.4:compile -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar:2.13.4:compile -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:2.13.4:compile -com.fasterxml.jackson.module:jackson-module-parameter-names:jar:2.13.4:compile -com.fasterxml.jackson.module:jackson-module-scala_2.13:jar:2.13.4:compile -com.github.ben-manes.caffeine:caffeine:jar:3.1.1:compile +com.fasterxml.jackson.core:jackson-annotations:jar:2.14.3:compile +com.fasterxml.jackson.core:jackson-core:jar:2.14.3:compile +com.fasterxml.jackson.core:jackson-databind:jar:2.14.3:compile +com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar:2.14.3:compile +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar:2.14.3:compile +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:2.14.3:compile +com.fasterxml.jackson.module:jackson-module-parameter-names:jar:2.14.3:compile +com.fasterxml.jackson.module:jackson-module-scala_2.13:jar:2.14.3:compile +com.github.ben-manes.caffeine:caffeine:jar:3.1.6:compile com.github.jnr:jffi:jar:1.2.18:compile com.github.jnr:jffi:jar:native:1.2.18:compile com.github.jnr:jnr-a64asm:jar:1.0.0:compile diff --git a/legal/3rd-party-dependencies/maven-plugins.txt b/legal/3rd-party-dependencies/maven-plugins.txt index 52967273523..4eeaf96c5b3 100644 --- a/legal/3rd-party-dependencies/maven-plugins.txt +++ b/legal/3rd-party-dependencies/maven-plugins.txt @@ -2,7 +2,7 @@ com.github.siom79.japicmp:japicmp-maven-plugin:maven-plugin:0.15.7:runtime com.mycila:license-maven-plugin:maven-plugin:4.1:runtime net.alchim31.maven:scala-maven-plugin:maven-plugin:4.5.6:runtime org.apache.felix:maven-bundle-plugin:maven-plugin:5.1.4:runtime -org.apache.maven.plugins:maven-clean-plugin:maven-plugin:2.5:runtime +org.apache.maven.plugins:maven-clean-plugin:maven-plugin:3.2.0:runtime org.apache.maven.plugins:maven-compiler-plugin:maven-plugin:3.9.0:runtime org.apache.maven.plugins:maven-dependency-plugin:maven-plugin:3.2.0:runtime org.apache.maven.plugins:maven-deploy-plugin:maven-plugin:3.0.0-M2:runtime @@ -14,7 +14,7 @@ org.apache.maven.plugins:maven-javadoc-plugin:maven-plugin:3.3.1:runtime org.apache.maven.plugins:maven-resources-plugin:maven-plugin:3.2.0:runtime org.apache.maven.plugins:maven-scm-plugin:maven-plugin:1.12.2:runtime org.apache.maven.plugins:maven-shade-plugin:maven-plugin:3.2.4:runtime -org.apache.maven.plugins:maven-site-plugin:maven-plugin:3.3:runtime +org.apache.maven.plugins:maven-site-plugin:maven-plugin:3.12.1:runtime org.apache.maven.plugins:maven-source-plugin:maven-plugin:3.2.1:runtime org.apache.maven.plugins:maven-surefire-plugin:maven-plugin:3.0.0-M5:runtime org.codehaus.mojo:flatten-maven-plugin:maven-plugin:1.2.7:runtime diff --git a/legal/NOTICE-THIRD-PARTY.md b/legal/NOTICE-THIRD-PARTY.md index 0deb8d7efea..c3f06bc3879 100644 --- a/legal/NOTICE-THIRD-PARTY.md +++ b/legal/NOTICE-THIRD-PARTY.md @@ -30,94 +30,94 @@ * Maven sources: https://search.maven.org/remotecontent?filepath=com/eclipsesource/minimal-json/minimal-json/0.9.5/minimal-json-0.9.5-sources.jar -## Jackson-annotations (2.13.4) +## Jackson-annotations (2.14.3) -* Maven coordinates: `com.fasterxml.jackson.core:jackson-annotations:2.13.4` +* Maven coordinates: `com.fasterxml.jackson.core:jackson-annotations:2.14.3` * License: [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) -* Project: http://github.com/FasterXML/jackson +* Project: https://github.com/FasterXML/jackson * Sources: * declared as SCM: https://github.com/FasterXML/jackson-annotations - * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/core/jackson-annotations/2.13.4/jackson-annotations-2.13.4-sources.jar + * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/core/jackson-annotations/2.14.3/jackson-annotations-2.14.3-sources.jar -## Jackson-core (2.13.4) +## Jackson-core (2.14.3) -* Maven coordinates: `com.fasterxml.jackson.core:jackson-core:2.13.4` +* Maven coordinates: `com.fasterxml.jackson.core:jackson-core:2.14.3` * License: [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) * Project: https://github.com/FasterXML/jackson-core * Sources: * declared as SCM: https://github.com/FasterXML/jackson-core - * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/core/jackson-core/2.13.4/jackson-core-2.13.4-sources.jar + * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/core/jackson-core/2.14.3/jackson-core-2.14.3-sources.jar -## jackson-databind (2.13.4.2) +## jackson-databind (2.14.3) -* Maven coordinates: `com.fasterxml.jackson.core:jackson-databind:2.13.4.2` +* Maven coordinates: `com.fasterxml.jackson.core:jackson-databind:2.14.3` * License: [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) -* Project: http://github.com/FasterXML/jackson +* Project: https://github.com/FasterXML/jackson * Sources: * declared as SCM: https://github.com/FasterXML/jackson-databind - * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/core/jackson-databind/2.13.4.2/jackson-databind-2.13.4.2-sources.jar + * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/core/jackson-databind/2.14.3/jackson-databind-2.14.3-sources.jar -## Jackson dataformat: CBOR (2.13.4) +## Jackson dataformat: CBOR (2.14.3) -* Maven coordinates: `com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.13.4` +* Maven coordinates: `com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.14.3` * License: [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) -* Project: http://github.com/FasterXML/jackson-dataformats-binary +* Project: https://github.com/FasterXML/jackson-dataformats-binary * Sources: * declared as SCM: https://github.com/FasterXML/jackson-dataformats-binary/jackson-dataformat-cbor - * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.13.4/jackson-dataformat-cbor-2.13.4-sources.jar + * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.14.3/jackson-dataformat-cbor-2.14.3-sources.jar -## Jackson datatype: jdk8 (2.13.4) +## Jackson datatype: jdk8 (2.14.3) -* Maven coordinates: `com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.4` +* Maven coordinates: `com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.14.3` * License: [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) * Project: https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jdk8 * Sources: * declared as SCM: https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jdk8 - * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/datatype/jackson-datatype-jdk8/2.13.4/jackson-datatype-jdk8-2.13.4-sources.jar + * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/datatype/jackson-datatype-jdk8/2.14.3/jackson-datatype-jdk8-2.14.3-sources.jar -## Jackson datatype: JSR310 (2.13.4) +## Jackson datatype: JSR310 (2.14.3) -* Maven coordinates: `com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.4` +* Maven coordinates: `com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.3` * License: [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) * Project: https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jsr310 * Sources: * declared as SCM: https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jsr310 - * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.13.4/jackson-datatype-jsr310-2.13.4-sources.jar + * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.14.3/jackson-datatype-jsr310-2.14.3-sources.jar -## Jackson-module-parameter-names (2.13.4) +## Jackson-module-parameter-names (2.14.3) -* Maven coordinates: `com.fasterxml.jackson.module:jackson-module-parameter-names:2.13.4` +* Maven coordinates: `com.fasterxml.jackson.module:jackson-module-parameter-names:2.14.3` * License: [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) * Project: https://github.com/FasterXML/jackson-modules-java8/jackson-module-parameter-names * Sources: * declared as SCM: https://github.com/FasterXML/jackson-modules-java8/jackson-module-parameter-names - * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/module/jackson-module-parameter-names/2.13.4/jackson-module-parameter-names-2.13.4-sources.jar + * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/module/jackson-module-parameter-names/2.14.3/jackson-module-parameter-names-2.14.3-sources.jar -## jackson-module-scala (2.13.4) +## jackson-module-scala (2.14.3) -* Maven coordinates: `com.fasterxml.jackson.module:jackson-module-scala_2.13:2.13.4` +* Maven coordinates: `com.fasterxml.jackson.module:jackson-module-scala_2.13:2.14.3` * License: [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) -* Project: http://wiki.fasterxml.com/JacksonModuleScala +* Project: https://github.com/FasterXML/jackson-module-scala * Sources: * declared as SCM: https://github.com/FasterXML/jackson-module-scala - * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/module/jackson-module-scala_2.13/2.13.4/jackson-module-scala_2.13-2.13.4-sources.jar + * Maven sources: https://search.maven.org/remotecontent?filepath=com/fasterxml/jackson/module/jackson-module-scala_2.13/2.14.3/jackson-module-scala_2.13-2.14.3-sources.jar -## Caffeine cache (3.1.1) +## Caffeine cache (3.1.6) -* Maven coordinates: `com.github.ben-manes.caffeine:caffeine:3.1.1` +* Maven coordinates: `com.github.ben-manes.caffeine:caffeine:3.1.6` * License: [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) * Project: https://github.com/ben-manes/caffeine * Sources: * declared as SCM: https://github.com/ben-manes/caffeine - * Maven sources: https://search.maven.org/remotecontent?filepath=com/github/ben-manes/caffeine/caffeine/3.1.1/caffeine-3.1.1-sources.jar + * Maven sources: https://search.maven.org/remotecontent?filepath=com/github/ben-manes/caffeine/caffeine/3.1.6/caffeine-3.1.6-sources.jar ## jffi (1.2.18) diff --git a/legal/NOTICE.md b/legal/NOTICE.md index 64523368e2d..8a361fe4b52 100644 --- a/legal/NOTICE.md +++ b/legal/NOTICE.md @@ -1,7 +1,7 @@ This content is produced and maintained by the Eclipse Ditto project. -* Project home: https://www.eclipse.org/ditto +* Project home: https://www.eclipse.dev/ditto/ # Trademarks @@ -24,7 +24,7 @@ SPDX-License-Identifier: EPL-2.0 # Source Code -* https://github.com/eclipse/ditto +* https://github.com/eclipse-ditto/ditto # Third-party Content diff --git a/messages/model/src/main/java/org/eclipse/ditto/messages/model/ThingIdInvalidException.java b/messages/model/src/main/java/org/eclipse/ditto/messages/model/ThingIdInvalidException.java index b5d830868ca..d80a817efa4 100755 --- a/messages/model/src/main/java/org/eclipse/ditto/messages/model/ThingIdInvalidException.java +++ b/messages/model/src/main/java/org/eclipse/ditto/messages/model/ThingIdInvalidException.java @@ -44,7 +44,7 @@ public final class ThingIdInvalidException extends EntityIdInvalidException impl "It must conform to the namespaced entity ID notation (see Ditto documentation)"; private static final URI DEFAULT_HREF = - URI.create("https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id"); + URI.create("https://www.eclipse.dev/ditto/basic-namespaces-and-names.html#namespaced-id"); private static final long serialVersionUID = -2426810319409279256L; diff --git a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/AbstractPolicyLoadingEnforcerActor.java b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/AbstractPolicyLoadingEnforcerActor.java index 023e6f0335d..18b3f44a438 100644 --- a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/AbstractPolicyLoadingEnforcerActor.java +++ b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/AbstractPolicyLoadingEnforcerActor.java @@ -34,7 +34,7 @@ public abstract class AbstractPolicyLoadingEnforcerActor, R extends CommandResponse, E extends EnforcementReloaded> extends AbstractEnforcerActor { - private final PolicyEnforcerProvider policyEnforcerProvider; + protected final PolicyEnforcerProvider policyEnforcerProvider; protected AbstractPolicyLoadingEnforcerActor(final I entityId, final E enforcement, diff --git a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/CachingPolicyEnforcerProvider.java b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/CachingPolicyEnforcerProvider.java index 9da753780d9..175b77d7a7a 100644 --- a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/CachingPolicyEnforcerProvider.java +++ b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/CachingPolicyEnforcerProvider.java @@ -19,6 +19,8 @@ import javax.annotation.Nullable; +import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.internal.utils.akka.logging.DittoDiagnosticLoggingAdapter; import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory; import org.eclipse.ditto.internal.utils.cache.config.CacheConfig; @@ -47,7 +49,7 @@ /** * Transparent caching layer for {@link org.eclipse.ditto.policies.enforcement.PolicyEnforcerProvider} */ -final class CachingPolicyEnforcerProvider extends AbstractPolicyEnforcerProvider { +final class CachingPolicyEnforcerProvider extends AbstractPolicyEnforcerProvider implements Invalidatable { private static final Logger LOGGER = DittoLoggerFactory.getThreadSafeLogger(CachingPolicyEnforcerProvider.class); private static final Duration LOCAL_POLICY_RETRIEVAL_TIMEOUT = Duration.ofSeconds(60); @@ -107,6 +109,22 @@ public CompletionStage> getPolicyEnforcer(@Nullable fin }); } + @Override + public CompletionStage invalidate(final PolicyTag policyTag, final String correlationId, + final Duration askTimeout) { + return Patterns.ask(cachingPolicyEnforcerProviderActor, new PolicyTagEnvelope(policyTag, correlationId), + askTimeout) + .thenApply(result -> { + if (result instanceof Boolean invalidated) { + return invalidated; + } + throw DittoInternalErrorException.fromMessage( + "Unexpected cachingPolicyEnforcerProviderActor response", + DittoHeaders.newBuilder().correlationId(correlationId).build()); + }); + } + + protected record PolicyTagEnvelope(PolicyTag policyTag, String correlationId){} /** * Actor which handles the actual cache lookup and invalidation. @@ -145,6 +163,12 @@ public Receive createReceive() { .match(PolicyId.class, this::doGetPolicyEnforcer) .match(DistributedPubSubMediator.SubscribeAck.class, s -> log.debug("Got subscribeAck <{}>.", s)) .match(PolicyTag.class, policyTag -> policyEnforcerCache.invalidate(policyTag.getEntityId())) + .match(PolicyTagEnvelope.class, policyTagEnvelope -> { + log.withCorrelationId(policyTagEnvelope.correlationId()).debug(policyTagEnvelope.correlationId()); + final boolean invalidated = + policyEnforcerCache.invalidate(policyTagEnvelope.policyTag().getEntityId()); + getSender().tell(invalidated, getSelf()); + }) .match(Replicator.Changed.class, this::handleChangedBlockedNamespaces) .build(); } diff --git a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/Invalidatable.java b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/Invalidatable.java new file mode 100644 index 00000000000..16f445802e5 --- /dev/null +++ b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/Invalidatable.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.ditto.policies.enforcement; + +import java.time.Duration; +import java.util.concurrent.CompletionStage; + +import org.eclipse.ditto.policies.api.PolicyTag; + +public interface Invalidatable { + + CompletionStage invalidate(PolicyTag policyTag, String correlationId, final Duration askTimeout); +} diff --git a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/config/DefaultCreationRestrictionConfig.java b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/config/DefaultCreationRestrictionConfig.java index 2714d017ce9..0ba36ad9f82 100644 --- a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/config/DefaultCreationRestrictionConfig.java +++ b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/config/DefaultCreationRestrictionConfig.java @@ -30,23 +30,27 @@ @Immutable public final class DefaultCreationRestrictionConfig implements CreationRestrictionConfig { - private static final String RESOURCE_TYPES_CONFIG_PATH = "resource-types"; - private static final String NAMESPACES_CONFIG_PATH = "namespaces"; - private static final String AUTH_SUBJECTS_CONFIG_PATH = "auth-subjects"; - private final Set resourceTypes; private final List namespacePatterns; private final List authSubjectPatterns; private DefaultCreationRestrictionConfig(final ConfigWithFallback configWithFallback) { - this.resourceTypes = Set.copyOf(configWithFallback.getStringList(RESOURCE_TYPES_CONFIG_PATH)); - this.namespacePatterns = compile(List.copyOf(configWithFallback.getStringList(NAMESPACES_CONFIG_PATH))); - this.authSubjectPatterns = compile(List.copyOf(configWithFallback.getStringList(AUTH_SUBJECTS_CONFIG_PATH))); + this.resourceTypes = Set.copyOf(configWithFallback.getStringList( + CreationRestrictionConfigValues.RESOURCE_TYPES.getConfigPath() + )); + this.namespacePatterns = compile(List.copyOf(configWithFallback.getStringList( + CreationRestrictionConfigValues.NAMESPACES.getConfigPath()) + )); + this.authSubjectPatterns = compile(List.copyOf(configWithFallback.getStringList( + CreationRestrictionConfigValues.AUTH_SUBJECTS.getConfigPath()) + )); } private static List compile(final List patterns) { return patterns.stream() - .map(expression -> Pattern.compile(LikeHelper.convertToRegexSyntax(expression))) + .map(LikeHelper::convertToRegexSyntax) + .filter(Objects::nonNull) + .map(Pattern::compile) .toList(); } diff --git a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/config/DefaultEntityCreationConfig.java b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/config/DefaultEntityCreationConfig.java index 2dcc2050f8a..64d9a652dca 100644 --- a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/config/DefaultEntityCreationConfig.java +++ b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/config/DefaultEntityCreationConfig.java @@ -28,7 +28,7 @@ @Immutable public final class DefaultEntityCreationConfig implements EntityCreationConfig { - private static final String CONFIG_PATH = "entity-creation"; + private static final String CONFIG_PATH = "ditto.entity-creation"; private final List grant; private final List revoke; diff --git a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/pre/CreationRestrictionPreEnforcer.java b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/pre/CreationRestrictionPreEnforcer.java index 67276ee1dd0..ad6e0c3b6cb 100644 --- a/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/pre/CreationRestrictionPreEnforcer.java +++ b/policies/enforcement/src/main/java/org/eclipse/ditto/policies/enforcement/pre/CreationRestrictionPreEnforcer.java @@ -55,7 +55,9 @@ public final class CreationRestrictionPreEnforcer implements PreEnforcer { */ @SuppressWarnings("unused") public CreationRestrictionPreEnforcer(final ActorSystem actorSystem, final Config config) { - this.config = DefaultEntityCreationConfig.of(config); + // explicitly use the ActorSystem config instead of the PreEnforcer config - as the config is loaded from + // file "ditto-entity-creation.conf" and extending with system properties of that file should not be broken + this.config = DefaultEntityCreationConfig.of(actorSystem.settings().config()); } boolean canCreate(final Context context) { diff --git a/policies/enforcement/src/test/java/org/eclipse/ditto/policies/enforcement/CachingPolicyEnforcerProviderTest.java b/policies/enforcement/src/test/java/org/eclipse/ditto/policies/enforcement/CachingPolicyEnforcerProviderTest.java index 7628b15b9e5..bd81f49db26 100644 --- a/policies/enforcement/src/test/java/org/eclipse/ditto/policies/enforcement/CachingPolicyEnforcerProviderTest.java +++ b/policies/enforcement/src/test/java/org/eclipse/ditto/policies/enforcement/CachingPolicyEnforcerProviderTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import java.time.Duration; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -75,6 +76,27 @@ public void tearDown() { actorSystem = null; } + @Test + public void callingInvalidateReturnsCachingActorResponse() { + final ActorSystem system = mock(ActorSystem.class); + when(system.actorOf(any())).thenReturn(cachingActorTestProbe.ref()); + final var underTest = new CachingPolicyEnforcerProvider( + system, + cache, + blockedNamespaces, + pubSubMediatorProbe.ref() + ); + + new TestKit(actorSystem) {{ + final var policyEnforcer = underTest.invalidate(PolicyTag.of(PolicyId.of("ns:id"), 1L), "correlationId", + Duration.ofSeconds(1L)); + cachingActorTestProbe.expectMsgClass(CachingPolicyEnforcerProvider.PolicyTagEnvelope.class); + cachingActorTestProbe.reply(true); + assertThat(policyEnforcer.toCompletableFuture().join()).isTrue(); + cachingActorTestProbe.expectNoMsg(); + }}; + } + @Test public void getPolicyEnforcerWithNullIdReturnsEmptyOptional() { final ActorSystem system = mock(ActorSystem.class); diff --git a/policies/enforcement/src/test/resources/entity-creation/default.conf b/policies/enforcement/src/test/resources/entity-creation/default.conf index 09e77540432..b5a6255acf2 100644 --- a/policies/enforcement/src/test/resources/entity-creation/default.conf +++ b/policies/enforcement/src/test/resources/entity-creation/default.conf @@ -1,4 +1,4 @@ -entity-creation { +ditto.entity-creation { grant = [{}] revoke = [] } diff --git a/policies/enforcement/src/test/resources/entity-creation/restricted1.conf b/policies/enforcement/src/test/resources/entity-creation/restricted1.conf index 49cff709b50..ff04466da27 100644 --- a/policies/enforcement/src/test/resources/entity-creation/restricted1.conf +++ b/policies/enforcement/src/test/resources/entity-creation/restricted1.conf @@ -1,4 +1,4 @@ -entity-creation { +ditto.entity-creation { grant = [ { resource-types = ["policy"] diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/ImmutablePolicyImports.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/ImmutablePolicyImports.java index 74bdcff47a7..3b32b0c971c 100644 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/ImmutablePolicyImports.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/ImmutablePolicyImports.java @@ -47,14 +47,14 @@ @Immutable final class ImmutablePolicyImports implements PolicyImports { - private final Map policyImports; + private final Map policyImports; public static final String POLICY_IMPORTS = "policyImports"; private ImmutablePolicyImports() { this.policyImports = Collections.emptyMap(); } - private ImmutablePolicyImports(final Map policyImports) { + private ImmutablePolicyImports(final Map policyImports) { checkNotNull(policyImports, POLICY_IMPORTS); this.policyImports = Collections.unmodifiableMap(new HashMap<>(policyImports)); } @@ -69,7 +69,7 @@ private ImmutablePolicyImports(final Map policyImpor public static ImmutablePolicyImports of(final Iterable policyImports) { checkNotNull(policyImports, POLICY_IMPORTS); - final Map resourcesMap = new HashMap<>(); + final Map resourcesMap = new HashMap<>(); policyImports.forEach(policyImport -> { final PolicyImport existingPolicyImport = resourcesMap.put(policyImport.getImportedPolicyId(), policyImport); @@ -113,7 +113,7 @@ public static PolicyImports fromJson(final JsonObject jsonObject) { @Override public Optional getPolicyImport(final CharSequence importedPolicyId) { checkNotNull(importedPolicyId, "importedPolicyId"); - return Optional.ofNullable(policyImports.get(importedPolicyId)); + return Optional.ofNullable(policyImports.get(PolicyId.of(importedPolicyId))); } @Override @@ -143,7 +143,7 @@ public PolicyImports setPolicyImports(final PolicyImports policyImports) { private PolicyImports createNewPolicyImportsWithNewPolicyImport(final PolicyImport newPolicyImport) { - final Map resourcesCopy = copyPolicyImports(); + final Map resourcesCopy = copyPolicyImports(); resourcesCopy.put(newPolicyImport.getImportedPolicyId(), newPolicyImport); if (resourcesCopy.size() > DITTO_LIMITS_POLICY_IMPORTS_LIMIT) { throw PolicyImportsTooLargeException.newBuilder(newPolicyImport.getImportedPolicyId()).build(); @@ -151,7 +151,7 @@ private PolicyImports createNewPolicyImportsWithNewPolicyImport(final PolicyImpo return new ImmutablePolicyImports(resourcesCopy); } - private Map copyPolicyImports() { + private Map copyPolicyImports() { return new HashMap<>(policyImports); } @@ -159,12 +159,13 @@ private Map copyPolicyImports() { public PolicyImports removePolicyImport(final CharSequence importedPolicyId) { checkNotNull(importedPolicyId, "importedPolicyId"); - if (!policyImports.containsKey(importedPolicyId)) { + final PolicyId policyId = PolicyId.of(importedPolicyId); + if (!policyImports.containsKey(policyId)) { return this; } - final Map resourcesCopy = copyPolicyImports(); - resourcesCopy.remove(importedPolicyId); + final Map resourcesCopy = copyPolicyImports(); + resourcesCopy.remove(policyId); return new ImmutablePolicyImports(resourcesCopy); } diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/PolicyIdInvalidException.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/PolicyIdInvalidException.java index 95b7351fdde..d7802c6037d 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/PolicyIdInvalidException.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/PolicyIdInvalidException.java @@ -44,7 +44,7 @@ public final class PolicyIdInvalidException extends EntityIdInvalidException imp private static final String DEFAULT_DESCRIPTION = "It must conform to the namespaced entity ID notation (see Ditto documentation)"; - private static final URI DEFAULT_HREF = URI.create("https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id"); + private static final URI DEFAULT_HREF = URI.create("https://www.eclipse.dev/ditto/basic-namespaces-and-names.html#namespaced-id"); private static final long serialVersionUID = 8154256308793903738L; diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/WithPolicyId.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/WithPolicyId.java new file mode 100644 index 00000000000..904115a7e43 --- /dev/null +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/WithPolicyId.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.policies.model; + +import org.eclipse.ditto.base.model.entity.id.WithEntityId; + +/** + * Implementations of this interface are associated to a {@code Policy} identified by the value + * returned from {@link #getEntityId()}. + * + * @since 3.2.0 + */ +public interface WithPolicyId extends WithEntityId { + + @Override + PolicyId getEntityId(); + +} diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java index 1ed450b33d4..c571ffdd52b 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java @@ -12,24 +12,26 @@ */ package org.eclipse.ditto.policies.model.signals.commands; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.base.model.entity.type.EntityType; import org.eclipse.ditto.base.model.entity.type.WithEntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.policies.model.PolicyConstants; -import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.policies.model.PolicyConstants; +import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.policies.model.WithPolicyId; /** * Aggregates all {@link Command}s which are related to a {@link org.eclipse.ditto.policies.model.Policy}. * * @param the type of the implementing class. */ -public interface PolicyCommand> extends Command, WithEntityType, SignalWithEntityId { +public interface PolicyCommand> extends Command, WithEntityType, WithPolicyId, + SignalWithEntityId { /** * Type Prefix of Policy commands. diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommandResponse.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommandResponse.java index 2933242fc64..16d6d489d46 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommandResponse.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommandResponse.java @@ -12,14 +12,15 @@ */ package org.eclipse.ditto.policies.model.signals.commands; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.policies.model.WithPolicyId; /** * Aggregates all possible responses relating to a given {@link PolicyCommand}. @@ -27,7 +28,7 @@ * @param the type of the implementing class. */ public interface PolicyCommandResponse> extends CommandResponse, - SignalWithEntityId { + WithPolicyId, SignalWithEntityId { /** * Type Prefix of Policy command responses. diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyHistoryNotAccessibleException.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyHistoryNotAccessibleException.java new file mode 100755 index 00000000000..51401af3718 --- /dev/null +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyHistoryNotAccessibleException.java @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.policies.model.signals.commands.exceptions; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.net.URI; +import java.text.MessageFormat; +import java.time.Instant; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.policies.model.PolicyException; +import org.eclipse.ditto.policies.model.PolicyId; + +/** + * Thrown if historical data of the Policy was either not present in Ditto at all or if the requester had insufficient + * permissions to access it. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableException(errorCode = PolicyHistoryNotAccessibleException.ERROR_CODE) +public final class PolicyHistoryNotAccessibleException extends DittoRuntimeException implements PolicyException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "policy.history.notfound"; + + private static final String MESSAGE_TEMPLATE = + "The Policy with ID ''{0}'' at revision ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String MESSAGE_TEMPLATE_TS = + "The Policy with ID ''{0}'' at timestamp ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String DEFAULT_DESCRIPTION = + "Check if the ID of your requested Policy was correct, you have sufficient permissions and ensure that the " + + "asked for revision/timestamp does not exceed the history-retention-duration."; + + private static final long serialVersionUID = 4242422323239998882L; + + private PolicyHistoryNotAccessibleException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + super(ERROR_CODE, HttpStatus.NOT_FOUND, dittoHeaders, message, description, cause, href); + } + + private static String getMessage(final PolicyId policyId, final long revision) { + checkNotNull(policyId, "policyId"); + return MessageFormat.format(MESSAGE_TEMPLATE, String.valueOf(policyId), String.valueOf(revision)); + } + + private static String getMessage(final PolicyId policyId, final Instant timestamp) { + checkNotNull(policyId, "policyId"); + checkNotNull(timestamp, "timestamp"); + return MessageFormat.format(MESSAGE_TEMPLATE_TS, String.valueOf(policyId), timestamp.toString()); + } + + /** + * A mutable builder for a {@code PolicyHistoryNotAccessibleException}. + * + * @param policyId the ID of the policy. + * @param revision the asked for revision of the policy. + * @return the builder. + * @throws NullPointerException if {@code policyId} is {@code null}. + */ + public static Builder newBuilder(final PolicyId policyId, final long revision) { + return new Builder(policyId, revision); + } + + /** + * A mutable builder for a {@code PolicyHistoryNotAccessibleException}. + * + * @param policyId the ID of the policy. + * @param timestamp the asked for timestamp of the policy. + * @return the builder. + * @throws NullPointerException if {@code policyId} is {@code null}. + */ + public static Builder newBuilder(final PolicyId policyId, final Instant timestamp) { + return new Builder(policyId, timestamp); + } + + /** + * Constructs a new {@code PolicyHistoryNotAccessibleException} object with given message. + * + * @param message detail message. This message can be later retrieved by the {@link #getMessage()} method. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new PolicyHistoryNotAccessibleException. + * @throws NullPointerException if {@code dittoHeaders} is {@code null}. + */ + public static PolicyHistoryNotAccessibleException fromMessage(@Nullable final String message, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromMessage(message, dittoHeaders, new Builder()); + } + + /** + * Constructs a new {@code PolicyHistoryNotAccessibleException} object with the exception message extracted from the given + * JSON object. + * + * @param jsonObject the JSON to read the {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new PolicyHistoryNotAccessibleException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static PolicyHistoryNotAccessibleException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyHistoryNotAccessibleException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() { + description(DEFAULT_DESCRIPTION); + } + + private Builder(final PolicyId policyId, final long revision) { + this(); + message(PolicyHistoryNotAccessibleException.getMessage(policyId, revision)); + } + + private Builder(final PolicyId policyId, final Instant timestamp) { + this(); + message(PolicyHistoryNotAccessibleException.getMessage(policyId, timestamp)); + } + + @Override + protected PolicyHistoryNotAccessibleException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new PolicyHistoryNotAccessibleException(dittoHeaders, message, description, cause, href); + } + + } + +} diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyPreconditionNotModifiedException.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyPreconditionNotModifiedException.java index 453e3c7c2fd..7e190e4c69f 100644 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyPreconditionNotModifiedException.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyPreconditionNotModifiedException.java @@ -19,12 +19,12 @@ import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.NotThreadSafe; -import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.base.model.common.HttpStatus; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.policies.model.PolicyException; @@ -58,7 +58,17 @@ private PolicyPreconditionNotModifiedException(final DittoHeaders dittoHeaders, } /** - * A mutable builder for a {@link org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyPreconditionNotModifiedException}. + * A mutable builder for a {@link PolicyPreconditionNotModifiedException}. + * + * @return the builder. + * @since 3.3.0 + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * A mutable builder for a {@link PolicyPreconditionNotModifiedException}. * * @param expectedNotToMatch the value which was expected not to match {@code matched} value. * @param matched the matched value. @@ -82,7 +92,7 @@ public static PolicyPreconditionNotModifiedException fromMessage(@Nullable final } /** - * Constructs a new {@link org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyPreconditionNotModifiedException} object with the exception message extracted from + * Constructs a new {@link PolicyPreconditionNotModifiedException} object with the exception message extracted from * the given JSON object. * * @param jsonObject the JSON to read the @@ -111,7 +121,7 @@ public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { } /** - * A mutable builder with a fluent API for a {@link org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyPreconditionNotModifiedException}. + * A mutable builder with a fluent API for a {@link PolicyPreconditionNotModifiedException}. */ @NotThreadSafe public static final class Builder extends DittoRuntimeExceptionBuilder { diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java index 0c956f8b7b5..6c5bf984dba 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java @@ -89,7 +89,7 @@ public static RetrievePolicy of(final PolicyId policyId, final DittoHeaders ditt * * @param policyId the ID of a single Policy to be retrieved by this command. * @param dittoHeaders the optional command headers of the request. - * @param selectedFields the fields of the JSON representation of the Thing to retrieve. + * @param selectedFields the fields of the JSON representation of the Policy to retrieve. * @return a Command for retrieving the Policy with the {@code policyId} as its ID which is readable from the passed * authorization context. * @throws NullPointerException if any argument is {@code null}. diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/AbstractPolicyEvent.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/AbstractPolicyEvent.java index 3e8cae99661..b94a0103f36 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/AbstractPolicyEvent.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/AbstractPolicyEvent.java @@ -20,8 +20,8 @@ import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.headers.DittoHeaders; -import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.base.model.signals.events.AbstractEventsourcedEvent; +import org.eclipse.ditto.policies.model.PolicyId; /** * Abstract base class of a {@link PolicyEvent}. @@ -57,6 +57,11 @@ protected AbstractPolicyEvent(final String type, this.policyId = policyId; } + @Override + public PolicyId getEntityId() { + return getPolicyEntityId(); + } + @Override public PolicyId getPolicyEntityId() { return policyId; diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java index c818ad61113..9e5c5a50132 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java @@ -14,21 +14,23 @@ import javax.annotation.concurrent.Immutable; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.policies.model.WithPolicyId; /** * Interface for all policy-related events. * * @param the type of the implementing class. */ -public interface PolicyEvent> extends EventsourcedEvent, SignalWithEntityId { +public interface PolicyEvent> extends EventsourcedEvent, WithPolicyId, + SignalWithEntityId { /** * Type Prefix of Policy events. diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsDeletedPartially.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsDeletedPartially.java index 190c8c10e5a..755acbe4696 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsDeletedPartially.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsDeletedPartially.java @@ -66,7 +66,7 @@ public final class SubjectsDeletedPartially extends AbstractPolicyActionEvent JSON_DELETED_SUBJECT_IDS = + public static final JsonFieldDefinition JSON_DELETED_SUBJECT_IDS = JsonFactory.newJsonObjectFieldDefinition("deletedSubjectIds", FieldType.REGULAR, JsonSchemaVersion.V_2); @@ -228,7 +228,13 @@ private static JsonObject deletedSubjectsToJson(final Map> deletedSubjectsFromJson(final JsonObject jsonObject) { + /** + * Transform the passed {@code jsonObject} to a map of deleted subjectIds as expected in the payload of this event. + * + * @param jsonObject the json object to read the modified subjects from. + * @return the map. + */ + public static Map> deletedSubjectsFromJson(final JsonObject jsonObject) { final Map> map = jsonObject.stream() .collect(Collectors.toMap(field -> Label.of(field.getKeyName()), field -> field.getValue().asArray().stream() diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsModifiedPartially.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsModifiedPartially.java index 3588333a549..a1c85153421 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsModifiedPartially.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsModifiedPartially.java @@ -67,7 +67,7 @@ public final class SubjectsModifiedPartially extends AbstractPolicyActionEvent JSON_MODIFIED_SUBJECTS = + public static final JsonFieldDefinition JSON_MODIFIED_SUBJECTS = JsonFactory.newJsonObjectFieldDefinition("modifiedSubjects", FieldType.REGULAR, JsonSchemaVersion.V_2); private final Map> modifiedSubjects; @@ -242,7 +242,13 @@ private static JsonObject modifiedSubjectsToJson(final Map> modifiedSubjectsFromJson(final JsonObject jsonObject) { + /** + * Transform the passed {@code jsonObject} to a map of modified subjects as expected in the payload of this event. + * + * @param jsonObject the json object to read the modified subjects from. + * @return the map. + */ + public static Map> modifiedSubjectsFromJson(final JsonObject jsonObject) { final Map> map = jsonObject.stream() .collect(Collectors.toMap(field -> Label.of(field.getKeyName()), field -> subjectsFromJsonWithId(field.getValue().asObject()), diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/DefaultPolicyConfig.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/DefaultPolicyConfig.java index fe125d9d2df..243773d0815 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/DefaultPolicyConfig.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/DefaultPolicyConfig.java @@ -23,7 +23,9 @@ import org.eclipse.ditto.internal.utils.config.ScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultActivityCheckConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultEventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultSnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.CleanupConfig; @@ -40,6 +42,7 @@ public final class DefaultPolicyConfig implements PolicyConfig { private final SupervisorConfig supervisorConfig; private final ActivityCheckConfig activityCheckConfig; private final SnapshotConfig snapshotConfig; + private final EventConfig eventConfig; private final Duration policySubjectExpiryGranularity; private final Duration policySubjectDeletionAnnouncementGranularity; private final String subjectIdResolver; @@ -50,6 +53,7 @@ private DefaultPolicyConfig(final ScopedConfig scopedConfig) { supervisorConfig = DefaultSupervisorConfig.of(scopedConfig); activityCheckConfig = DefaultActivityCheckConfig.of(scopedConfig); snapshotConfig = DefaultSnapshotConfig.of(scopedConfig); + eventConfig = DefaultEventConfig.of(scopedConfig); policySubjectExpiryGranularity = scopedConfig.getNonNegativeDurationOrThrow(PolicyConfigValue.SUBJECT_EXPIRY_GRANULARITY); policySubjectDeletionAnnouncementGranularity = @@ -89,6 +93,11 @@ public SnapshotConfig getSnapshotConfig() { return snapshotConfig; } + @Override + public EventConfig getEventConfig() { + return eventConfig; + } + @Override public Duration getSubjectExpiryGranularity() { return policySubjectExpiryGranularity; @@ -126,6 +135,7 @@ public boolean equals(final Object o) { return Objects.equals(supervisorConfig, that.supervisorConfig) && Objects.equals(activityCheckConfig, that.activityCheckConfig) && Objects.equals(snapshotConfig, that.snapshotConfig) && + Objects.equals(eventConfig, that.eventConfig) && Objects.equals(policySubjectExpiryGranularity, that.policySubjectExpiryGranularity) && Objects.equals(policySubjectDeletionAnnouncementGranularity, that.policySubjectDeletionAnnouncementGranularity) && @@ -136,9 +146,9 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(supervisorConfig, activityCheckConfig, snapshotConfig, policySubjectExpiryGranularity, - policySubjectDeletionAnnouncementGranularity, subjectIdResolver, policyAnnouncementConfig, - cleanupConfig); + return Objects.hash(supervisorConfig, activityCheckConfig, snapshotConfig, eventConfig, + policySubjectExpiryGranularity, policySubjectDeletionAnnouncementGranularity, subjectIdResolver, + policyAnnouncementConfig, cleanupConfig); } @Override @@ -147,6 +157,7 @@ public String toString() { " supervisorConfig=" + supervisorConfig + ", activityCheckConfig=" + activityCheckConfig + ", snapshotConfig=" + snapshotConfig + + ", eventConfig=" + eventConfig + ", policySubjectExpiryGranularity=" + policySubjectExpiryGranularity + ", policySubjectDeletionAnnouncementGranularity=" + policySubjectDeletionAnnouncementGranularity + ", subjectIdResolver=" + subjectIdResolver + diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/PolicyConfig.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/PolicyConfig.java index 83bc6fb910a..7c1c5755d26 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/PolicyConfig.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/PolicyConfig.java @@ -18,6 +18,7 @@ import org.eclipse.ditto.base.service.config.supervision.WithSupervisorConfig; import org.eclipse.ditto.internal.utils.config.KnownConfigValue; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithSnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.WithCleanupConfig; @@ -29,6 +30,13 @@ public interface PolicyConfig extends WithSupervisorConfig, WithActivityCheckConfig, WithSnapshotConfig, WithCleanupConfig { + /** + * Returns the config of the policy event journal behaviour. + * + * @return the config. + */ + EventConfig getEventConfig(); + /** * Returns the configuration to which duration the {@code expiry} of a {@code Policy Subject} should be rounded up * to. diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcement.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcement.java index 4a92b139c3e..488d41efa20 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcement.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcement.java @@ -18,7 +18,10 @@ import java.util.concurrent.CompletionStage; import org.eclipse.ditto.base.model.auth.AuthorizationContext; +import org.eclipse.ditto.base.model.entity.id.WithEntityId; +import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandToExceptionRegistry; import org.eclipse.ditto.json.JsonFactory; @@ -34,6 +37,7 @@ import org.eclipse.ditto.policies.model.PoliciesResourceType; import org.eclipse.ditto.policies.model.Policy; import org.eclipse.ditto.policies.model.PolicyEntry; +import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.policies.model.ResourceKey; import org.eclipse.ditto.policies.model.enforcers.Enforcer; import org.eclipse.ditto.policies.model.signals.commands.PolicyCommand; @@ -52,7 +56,7 @@ * Authorizes {@link PolicyCommand}s and filters {@link PolicyCommandResponse}s. */ public final class PolicyCommandEnforcement - extends AbstractEnforcementReloaded, PolicyCommandResponse> { + extends AbstractEnforcementReloaded, PolicyCommandResponse> { /** * Json fields that are always shown regardless of authorization. @@ -61,37 +65,38 @@ public final class PolicyCommandEnforcement JsonFactory.newFieldSelector(Policy.JsonFields.ID); @Override - public CompletionStage> authorizeSignal(final PolicyCommand command, + public CompletionStage> authorizeSignal(final Signal signal, final PolicyEnforcer policyEnforcer) { - if (command.getCategory() == Command.Category.QUERY && !command.getDittoHeaders().isResponseRequired()) { + if (signal instanceof Command command && + command.getCategory() == Command.Category.QUERY && !command.getDittoHeaders().isResponseRequired()) { // ignore query command with response-required=false return CompletableFuture.completedStage(null); } final Enforcer enforcer = policyEnforcer.getEnforcer(); - final var policyResourceKey = PoliciesResourceType.policyResource(command.getResourcePath()); - final var authorizationContext = command.getDittoHeaders().getAuthorizationContext(); - final PolicyCommand authorizedCommand; - if (command instanceof CreatePolicy createPolicy) { + final var policyResourceKey = PoliciesResourceType.policyResource(signal.getResourcePath()); + final var authorizationContext = signal.getDittoHeaders().getAuthorizationContext(); + final Signal authorizedCommand; + if (signal instanceof CreatePolicy createPolicy) { authorizedCommand = authorizeCreatePolicy(enforcer, createPolicy, policyResourceKey, authorizationContext); - } else if (command instanceof PolicyActionCommand) { - authorizedCommand = authorizeActionCommand(policyEnforcer, command, policyResourceKey, - authorizationContext).orElseThrow(() -> errorForPolicyCommand(command)); - } else if (command instanceof PolicyModifyCommand) { + } else if (signal instanceof PolicyActionCommand) { + authorizedCommand = authorizeActionCommand(policyEnforcer, signal, policyResourceKey, + authorizationContext).orElseThrow(() -> errorForPolicyCommand(signal)); + } else if (signal instanceof PolicyModifyCommand) { if (hasUnrestrictedWritePermission(enforcer, policyResourceKey, authorizationContext)) { - authorizedCommand = command; + authorizedCommand = signal; } else { - throw errorForPolicyCommand(command); + throw errorForPolicyCommand(signal); } } else { final String permission = Permission.READ; if (enforcer.hasPartialPermissions(policyResourceKey, authorizationContext, permission)) { - authorizedCommand = command; + authorizedCommand = signal; } else { - throw errorForPolicyCommand(command); + throw errorForPolicyCommand(signal); } } @@ -112,8 +117,17 @@ private PolicyCommand authorizeCreatePolicy(final Enforcer enforcer, } @Override - public CompletionStage> authorizeSignalWithMissingEnforcer(final PolicyCommand command) { - throw PolicyNotAccessibleException.newBuilder(command.getEntityId()) + public CompletionStage> authorizeSignalWithMissingEnforcer(final Signal command) { + final PolicyId policyId; + if (command instanceof WithEntityId withEntityId) { + policyId = PolicyId.of(withEntityId.getEntityId()); + } else { + LOGGER.warn("Processed signal which does not have an entityId: {}", command); + throw DittoInternalErrorException.newBuilder() + .dittoHeaders(command.getDittoHeaders()) + .build(); + } + throw PolicyNotAccessibleException.newBuilder(policyId) .dittoHeaders(command.getDittoHeaders()) .build(); } @@ -143,7 +157,7 @@ public CompletionStage> filterResponse(final PolicyComm } @SuppressWarnings("unchecked") - private > Optional authorizeActionCommand( + private > Optional authorizeActionCommand( final PolicyEnforcer enforcer, final T command, final ResourceKey resourceKey, final AuthorizationContext authorizationContext) { @@ -154,7 +168,7 @@ private > Optional authorizeActionCommand( } } - private > Optional authorizeEntryLevelAction(final Enforcer enforcer, + private > Optional authorizeEntryLevelAction(final Enforcer enforcer, final T command, final ResourceKey resourceKey, final AuthorizationContext authorizationContext) { return enforcer.hasUnrestrictedPermissions(resourceKey, authorizationContext, Permission.EXECUTE) ? Optional.of(command) : Optional.empty(); @@ -232,19 +246,32 @@ private JsonObject getJsonViewForPolicyQueryCommandResponse(final JsonObject res /** * Create error due to failing to execute a policy-command in the expected way. * - * @param policyCommand the command. + * @param policySignal the signal. * @return the error. */ - private static DittoRuntimeException errorForPolicyCommand(final PolicyCommand policyCommand) { - final CommandToExceptionRegistry, DittoRuntimeException> registry; - if (policyCommand instanceof PolicyActionCommand) { - registry = PolicyCommandToActionsExceptionRegistry.getInstance(); - } else if (policyCommand instanceof PolicyModifyCommand) { - registry = PolicyCommandToModifyExceptionRegistry.getInstance(); + private static DittoRuntimeException errorForPolicyCommand(final Signal policySignal) { + + if (policySignal instanceof PolicyCommand policyCommand) { + final CommandToExceptionRegistry, DittoRuntimeException> registry; + if (policyCommand instanceof PolicyActionCommand) { + registry = PolicyCommandToActionsExceptionRegistry.getInstance(); + } else if (policyCommand instanceof PolicyModifyCommand) { + registry = PolicyCommandToModifyExceptionRegistry.getInstance(); + } else { + registry = PolicyCommandToAccessExceptionRegistry.getInstance(); + } + return registry.exceptionFrom(policyCommand); + } else if (policySignal instanceof WithEntityId withEntityId) { + return PolicyNotAccessibleException.newBuilder(PolicyId.of(withEntityId.getEntityId())) + .dittoHeaders(policySignal.getDittoHeaders()) + .build(); } else { - registry = PolicyCommandToAccessExceptionRegistry.getInstance(); + LOGGER.error("Received signal for which no DittoRuntimeException due to lack of access " + + "could be determined: {}", policySignal); + return DittoInternalErrorException.newBuilder() + .dittoHeaders(policySignal.getDittoHeaders()) + .build(); } - return registry.exceptionFrom(policyCommand); } /** diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyEnforcerActor.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyEnforcerActor.java index 4b96dd1a42c..8830ec46251 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyEnforcerActor.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyEnforcerActor.java @@ -37,7 +37,7 @@ * {@link PolicyCommandEnforcement}. */ public final class PolicyEnforcerActor extends - AbstractPolicyLoadingEnforcerActor, PolicyCommandResponse, PolicyCommandEnforcement> { + AbstractPolicyLoadingEnforcerActor, PolicyCommandResponse, PolicyCommandEnforcement> { private static final String ENFORCEMENT_DISPATCHER = "enforcement-dispatcher"; diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActor.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActor.java index a64f0c3bb52..3fbaa4a7579 100755 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActor.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActor.java @@ -12,7 +12,9 @@ */ package org.eclipse.ditto.policies.service.persistence.actors; +import java.time.Instant; import java.util.Set; +import java.util.concurrent.CompletionStage; import java.util.stream.StreamSupport; import javax.annotation.Nullable; @@ -26,6 +28,7 @@ import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceActor; import org.eclipse.ditto.internal.utils.persistentactors.commands.CommandStrategy; import org.eclipse.ditto.internal.utils.persistentactors.commands.DefaultContext; @@ -36,6 +39,7 @@ import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.policies.model.PolicyLifecycle; import org.eclipse.ditto.policies.model.Subjects; +import org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyHistoryNotAccessibleException; import org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyNotAccessibleException; import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.policies.service.common.config.DittoPoliciesConfig; @@ -76,11 +80,12 @@ public final class PolicyPersistenceActor @SuppressWarnings("unused") private PolicyPersistenceActor(final PolicyId policyId, + final MongoReadJournal mongoReadJournal, final ActorRef pubSubMediator, final ActorRef announcementManager, final PolicyConfig policyConfig) { - super(policyId); + super(policyId, mongoReadJournal); this.pubSubMediator = pubSubMediator; this.announcementManager = announcementManager; this.policyConfig = policyConfig; @@ -88,12 +93,13 @@ private PolicyPersistenceActor(final PolicyId policyId, } private PolicyPersistenceActor(final PolicyId policyId, + final MongoReadJournal mongoReadJournal, final ActorRef pubSubMediator, final ActorRef announcementManager, final ActorRef supervisor) { // not possible to call other constructor because "getContext()" is not available as argument of "this()" - super(policyId); + super(policyId, mongoReadJournal); this.pubSubMediator = pubSubMediator; this.announcementManager = announcementManager; this.supervisor = supervisor; @@ -107,25 +113,30 @@ private PolicyPersistenceActor(final PolicyId policyId, * Creates Akka configuration object {@link Props} for this PolicyPersistenceActor. * * @param policyId the ID of the Policy this Actor manages. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the policy. * @param pubSubMediator the PubSub mediator actor. * @param announcementManager manager of policy announcements. * @param policyConfig the policy config. * @return the Akka configuration Props object */ public static Props props(final PolicyId policyId, + final MongoReadJournal mongoReadJournal, final ActorRef pubSubMediator, final ActorRef announcementManager, final PolicyConfig policyConfig) { - return Props.create(PolicyPersistenceActor.class, policyId, pubSubMediator, announcementManager, policyConfig); + return Props.create(PolicyPersistenceActor.class, policyId, mongoReadJournal, pubSubMediator, + announcementManager, policyConfig); } static Props propsForTests(final PolicyId policyId, + final MongoReadJournal mongoReadJournal, final ActorRef pubSubMediator, final ActorRef announcementManager, final ActorSystem actorSystem) { - return Props.create(PolicyPersistenceActor.class, policyId, pubSubMediator, announcementManager, + return Props.create(PolicyPersistenceActor.class, policyId, mongoReadJournal, pubSubMediator, + announcementManager, actorSystem.deadLetters()); } @@ -189,6 +200,16 @@ protected DittoRuntimeExceptionBuilder newNotAccessibleExceptionBuilder() { return PolicyNotAccessibleException.newBuilder(entityId); } + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final long revision) { + return PolicyHistoryNotAccessibleException.newBuilder(entityId, revision); + } + + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final Instant timestamp) { + return PolicyHistoryNotAccessibleException.newBuilder(entityId, timestamp); + } + @Override protected void publishEvent(@Nullable final Policy previousEntity, final PolicyEvent event) { @@ -228,9 +249,29 @@ private void publishPolicyTag(final PolicyEvent event) { public void onMutation(final Command command, final PolicyEvent event, final WithDittoHeaders response, final boolean becomeCreated, final boolean becomeDeleted) { + final ActorRef sender = getSender(); persistAndApplyEvent(event, (persistedEvent, resultingEntity) -> { if (shouldSendResponse(command.getDittoHeaders())) { - notifySender(getSender(), response); + notifySender(sender, response); + } + if (becomeDeleted) { + becomeDeletedHandler(); + } + if (becomeCreated) { + becomeCreatedHandler(); + } + }); + } + + @Override + public void onStagedMutation(final Command command, final CompletionStage> event, + final CompletionStage response, + final boolean becomeCreated, final boolean becomeDeleted) { + + final ActorRef sender = getSender(); + persistAndApplyEventAsync(event, (persistedEvent, resultingEntity) -> { + if (shouldSendResponse(command.getDittoHeaders())) { + response.thenAccept(rsp -> notifySender(sender, rsp)); } if (becomeDeleted) { becomeDeletedHandler(); diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActor.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActor.java index bf5869071e2..b958bc9ac76 100755 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActor.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActor.java @@ -20,8 +20,10 @@ import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; import org.eclipse.ditto.base.service.actors.ShutdownBehaviour; import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOffConfig; +import org.eclipse.ditto.base.service.config.supervision.LocalAskTimeoutConfig; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.policies.enforcement.PolicyEnforcerProvider; @@ -58,9 +60,10 @@ public final class PolicySupervisorActor extends AbstractPersistenceSupervisor

    > policyAnnouncementPub, @Nullable final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { - super(blockedNamespaces, DEFAULT_LOCAL_ASK_TIMEOUT); + super(blockedNamespaces, mongoReadJournal); this.policyEnforcerProvider = policyEnforcerProvider; this.pubSubMediator = pubSubMediator; final DittoPoliciesConfig policiesConfig = DittoPoliciesConfig.of( @@ -88,15 +91,17 @@ private PolicySupervisorActor(final ActorRef pubSubMediator, * @param policyAnnouncementPub publisher interface of policy announcements. * @param blockedNamespaces the blocked namespaces functionality to retrieve/subscribe for blocked namespaces. * @param policyEnforcerProvider used to load the policy enforcer to authorize commands. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the policy. * @return the {@link Props} to create this actor. */ public static Props props(final ActorRef pubSubMediator, final DistributedPub> policyAnnouncementPub, @Nullable final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { - return Props.create(PolicySupervisorActor.class, () -> new PolicySupervisorActor(pubSubMediator, - policyAnnouncementPub, blockedNamespaces, policyEnforcerProvider)); + return Props.create(PolicySupervisorActor.class, pubSubMediator, + policyAnnouncementPub, blockedNamespaces, policyEnforcerProvider, mongoReadJournal); } @Override @@ -106,7 +111,8 @@ protected PolicyId getEntityId() throws Exception { @Override protected Props getPersistenceActorProps(final PolicyId entityId) { - return PolicyPersistenceActor.props(entityId, pubSubMediator, announcementManager, policyConfig); + return PolicyPersistenceActor.props(entityId, mongoReadJournal, pubSubMediator, announcementManager, + policyConfig); } @Override @@ -123,6 +129,14 @@ protected ExponentialBackOffConfig getExponentialBackOffConfig() { return policiesConfig.getPolicyConfig().getSupervisorConfig().getExponentialBackOffConfig(); } + @Override + protected LocalAskTimeoutConfig getLocalAskTimeoutConfig() { + return DittoPoliciesConfig.of(DefaultScopedConfig.dittoScoped(getContext().getSystem().settings().config())) + .getPolicyConfig() + .getSupervisorConfig() + .getLocalAskTimeoutConfig(); + } + @Override protected ShutdownBehaviour getShutdownBehaviour(final PolicyId entityId) { return ShutdownBehaviour.fromId(entityId, pubSubMediator, getSelf()); diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PoliciesConditionalHeadersValidatorProvider.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PoliciesConditionalHeadersValidatorProvider.java index b5b65142a3e..158cbadf57a 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PoliciesConditionalHeadersValidatorProvider.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PoliciesConditionalHeadersValidatorProvider.java @@ -56,6 +56,13 @@ public DittoRuntimeExceptionBuilder createPreconditionNotModifiedExceptionBui final String expectedNotToMatch, final String matched) { return PolicyPreconditionNotModifiedException.newBuilder(expectedNotToMatch, matched); } + + @Override + public DittoRuntimeExceptionBuilder createPreconditionNotModifiedForEqualityExceptionBuilder() { + return PolicyPreconditionNotModifiedException.newBuilder() + .message("The previous value was equal to the new value and the 'if-equal' header was set to 'skip'.") + .description("Your changes were not applied, which is probably the expected outcome."); + } } private static final ConditionalHeadersValidator INSTANCE = createInstance(); diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/TopLevelPolicyActionCommandStrategy.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/TopLevelPolicyActionCommandStrategy.java index d5e82708ff9..14f12ad766a 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/TopLevelPolicyActionCommandStrategy.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/TopLevelPolicyActionCommandStrategy.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletionStage; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -174,7 +175,8 @@ private Optional> aggregateEvents() { @Override public void onMutation(final Command command, final PolicyActionEvent event, - final WithDittoHeaders response, final boolean becomeCreated, final boolean becomeDeleted) { + final WithDittoHeaders response, final boolean becomeCreated, + final boolean becomeDeleted) { if (firstEvent == null) { firstEvent = event; } else { @@ -182,11 +184,27 @@ public void onMutation(final Command command, final PolicyActionEvent even } } + @Override + public void onStagedMutation(final Command command, final CompletionStage> event, + final CompletionStage response, final boolean becomeCreated, + final boolean becomeDeleted) { + if (firstEvent == null) { + firstEvent = event.toCompletableFuture().join(); + } else { + otherEvents.add(event.thenApply(x -> (PolicyActionEvent) x).toCompletableFuture().join()); + } + } + @Override public void onQuery(final Command command, final WithDittoHeaders response) { // do nothing } + @Override + public void onStagedQuery(final Command command, final CompletionStage response) { + // do nothing + } + @Override public void onError(final DittoRuntimeException error, final Command errorCausingCommand) { this.error = error; diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/events/PolicyEventStrategies.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/events/PolicyEventStrategies.java index 10afaeddbec..18fae3502a5 100755 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/events/PolicyEventStrategies.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/events/PolicyEventStrategies.java @@ -38,7 +38,7 @@ import org.eclipse.ditto.policies.model.signals.events.SubjectsModifiedPartially; /** - * Holds all {@link org.eclipse.ditto.policies.model.signals.events.PolicyEvent} strategies. + * Holds all {@link PolicyEvent} strategies. */ public final class PolicyEventStrategies extends AbstractEventStrategies, Policy> { diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/AbstractPolicyMongoEventAdapter.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/AbstractPolicyMongoEventAdapter.java index 822b0926561..b296cdb1a62 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/AbstractPolicyMongoEventAdapter.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/AbstractPolicyMongoEventAdapter.java @@ -13,16 +13,17 @@ package org.eclipse.ditto.policies.service.persistence.serializer; -import javax.annotation.Nullable; - -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; -import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry; +import org.eclipse.ditto.base.service.config.DittoServiceConfig; +import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.policies.service.common.config.DefaultPolicyConfig; import akka.actor.ExtendedActorSystem; @@ -36,8 +37,10 @@ public abstract class AbstractPolicyMongoEventAdapter extends AbstractMongoEvent JsonFactory.newJsonObjectFieldDefinition("policy/entries", FieldType.SPECIAL, JsonSchemaVersion.V_2); - protected AbstractPolicyMongoEventAdapter(@Nullable final ExtendedActorSystem system) { - super(system, GlobalEventRegistry.getInstance()); + protected AbstractPolicyMongoEventAdapter(final ExtendedActorSystem system) { + super(system, GlobalEventRegistry.getInstance(), DefaultPolicyConfig.of( + DittoServiceConfig.of(DefaultScopedConfig.dittoScoped(system.settings().config()), "policies")) + .getEventConfig()); } } diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/DefaultPolicyMongoEventAdapter.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/DefaultPolicyMongoEventAdapter.java index 59a0c3a1b19..ec5fec64cd5 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/DefaultPolicyMongoEventAdapter.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/DefaultPolicyMongoEventAdapter.java @@ -12,8 +12,6 @@ */ package org.eclipse.ditto.policies.service.persistence.serializer; -import javax.annotation.Nullable; - import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import akka.actor.ExtendedActorSystem; @@ -24,7 +22,7 @@ */ public final class DefaultPolicyMongoEventAdapter extends AbstractPolicyMongoEventAdapter { - public DefaultPolicyMongoEventAdapter(@Nullable final ExtendedActorSystem system) { + public DefaultPolicyMongoEventAdapter(final ExtendedActorSystem system) { super(system); } diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/starter/PoliciesRootActor.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/starter/PoliciesRootActor.java index 5db4b1f3105..f1ea2413c21 100755 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/starter/PoliciesRootActor.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/starter/PoliciesRootActor.java @@ -88,15 +88,16 @@ private PoliciesRootActor(final PoliciesConfig policiesConfig, final ActorRef pu BlockedNamespacesUpdater.ACTOR_NAME, blockedNamespacesUpdaterProps); final PolicyEnforcerProvider policyEnforcerProvider = PolicyEnforcerProviderExtension.get(actorSystem).getPolicyEnforcerProvider(); + final var mongoReadJournal = MongoReadJournal.newInstance(actorSystem); + final var policySupervisorProps = getPolicySupervisorActorProps(pubSubMediator, policyAnnouncementPub, blockedNamespaces, - policyEnforcerProvider); + policyEnforcerProvider, mongoReadJournal); final ActorRef policiesShardRegion = ShardRegionCreator.start(actorSystem, PoliciesMessagingConstants.SHARD_REGION, policySupervisorProps, policiesConfig.getClusterConfig().getNumberOfShards(), CLUSTER_ROLE); - final var mongoReadJournal = MongoReadJournal.newInstance(actorSystem); startClusterSingletonActor( PersistencePingActor.props(policiesShardRegion, policiesConfig.getPingConfig(), mongoReadJournal), PersistencePingActor.ACTOR_NAME); @@ -133,10 +134,11 @@ private PoliciesRootActor(final PoliciesConfig policiesConfig, final ActorRef pu private static Props getPolicySupervisorActorProps(final ActorRef pubSubMediator, final DistributedPub> policyAnnouncementPub, final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { return PolicySupervisorActor.props(pubSubMediator, policyAnnouncementPub, blockedNamespaces, - policyEnforcerProvider); + policyEnforcerProvider, mongoReadJournal); } /** diff --git a/policies/service/src/main/resources/policies.conf b/policies/service/src/main/resources/policies.conf index 0023efa356b..ea441dfee6d 100755 --- a/policies/service/src/main/resources/policies.conf +++ b/policies/service/src/main/resources/policies.conf @@ -65,6 +65,16 @@ ditto { threshold = ${?POLICY_SNAPSHOT_THRESHOLD} # may be overridden with this environment variable } + event { + # define the DittoHeaders to persist when persisting events to the journal + # those can e.g. be retrieved as additional "audit log" information when accessing a historical policy revision + historical-headers-to-persist = [ + #"ditto-originator" # who (user-subject/connection-pre-auth-subject) issued the event + #"correlation-id" + ] + historical-headers-to-persist = ${?POLICY_EVENT_HISTORICAL_HEADERS_TO_PERSIST} + } + supervisor { exponential-backoff { min = 1s @@ -102,27 +112,62 @@ ditto { } cleanup { + # enabled configures whether background cleanup is enabled or not + # If enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process enabled = true enabled = ${?CLEANUP_ENABLED} + # history-retention-duration configures the duration of how long to "keep" events and snapshots before being + # allowed to remove them in scope of cleanup. + # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read + # journal. + history-retention-duration = 30d + history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} + + # quiet-period defines how long to stay in a state where the background cleanup is not yet started + # Applies after: + # - starting the service + # - each "completed" background cleanup run (all entities were cleaned up) quiet-period = 5m quiet-period = ${?CLEANUP_QUIET_PERIOD} + # interval configures how often a "credit decision" is made. + # The background cleanup works with a credit system and does only generate new "cleanup credits" if the MongoDB + # currently has capacity to do cleanups. interval = 10s interval = ${?CLEANUP_INTERVAL} + # timer-threshold configures the maximum database latency to give out credit for cleanup actions. + # If write operations to the MongoDB within the last `interval` had a `max` value greater to the configured + # threshold, no new cleanup credits will be issued for the next `interval`. + # Which throttles cleanup when MongoDB is currently under heavy (write) load. timer-threshold = 150ms timer-threshold = ${?CLEANUP_TIMER_THRESHOLD} + # credits-per-batch configures how many "cleanup credits" should be generated per `interval` as long as the + # write operations to the MongoDB are less than the configured `timer-threshold`. + # Limits the rate of cleanup actions to this many per credit decision interval. + # One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`. credits-per-batch = 3 credits-per-batch = ${?CLEANUP_CREDITS_PER_BATCH} + # reads-per-query configures the number of snapshots to scan per MongoDB query. + # Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned + # with the amount of "cleanup credits" issued per `interval` - in order to avoid long running queries. reads-per-query = 100 reads-per-query = ${?CLEANUP_READS_PER_QUERY} + # writes-per-credit configures the number of documents to delete for each credit. + # If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead + # to 10 delete operations performed against MongoDB. writes-per-credit = 100 writes-per-credit = ${?CLEANUP_WRITES_PER_CREDIT} + # delete-final-deleted-snapshot configures whether for a deleted entity, the final snapshot (containing the + # "deleted" information) should be deleted or not. + # If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with + # a revision number 1 higher than the previously deleted entity. If the final snapshot is deleted as well, + # recreation of an entity with the same ID will lead to revisionNumber=1 after its recreation. delete-final-deleted-snapshot = false delete-final-deleted-snapshot = ${?CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT} } @@ -219,6 +264,18 @@ akka-contrib-mongodb-persistence-policies-journal { } } +akka-contrib-mongodb-persistence-policies-journal-read { + class = "akka.contrib.persistence.mongodb.MongoReadJournal" + plugin-dispatcher = "policy-journal-persistence-dispatcher" + + overrides { + journal-collection = "policies_journal" + journal-index = "policies_journal_index" + realtime-collection = "policies_realtime" + metadata-collection = "policies_metadata" + } +} + akka-contrib-mongodb-persistence-policies-snapshots { class = "akka.contrib.persistence.mongodb.MongoSnapshots" plugin-dispatcher = "policy-snaps-persistence-dispatcher" diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcementTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcementTest.java index 2f367ad8b55..fcb75ae0ce7 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcementTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcementTest.java @@ -17,7 +17,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -41,7 +40,9 @@ import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.service.actors.ShutdownBehaviour; import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOffConfig; +import org.eclipse.ditto.base.service.config.supervision.LocalAskTimeoutConfig; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.json.JsonObject; @@ -785,7 +786,7 @@ private class MockPolicyPersistenceSupervisor private MockPolicyPersistenceSupervisor(final ActorRef pubSubMediator, final ActorRef policyPersistenceActor, final PolicyEnforcerProvider policyEnforcerProvider) { - super(policyPersistenceActor, null, null, Duration.ofSeconds(5)); + super(policyPersistenceActor, null, null, Mockito.mock(MongoReadJournal.class)); this.pubSubMediator = pubSubMediator; this.policyEnforcerProvider = policyEnforcerProvider; } @@ -813,6 +814,14 @@ protected ExponentialBackOffConfig getExponentialBackOffConfig() { return policiesConfig.getPolicyConfig().getSupervisorConfig().getExponentialBackOffConfig(); } + @Override + protected LocalAskTimeoutConfig getLocalAskTimeoutConfig() { + return DittoPoliciesConfig.of(DefaultScopedConfig.dittoScoped(getContext().getSystem().settings().config())) + .getPolicyConfig() + .getSupervisorConfig() + .getLocalAskTimeoutConfig(); + } + @Override protected ShutdownBehaviour getShutdownBehaviour(final PolicyId entityId) { return ShutdownBehaviour.fromId(entityId, pubSubMediator, getSelf()); diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorSnapshottingTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorSnapshottingTest.java index dc0369b511f..10ccdd21ee3 100755 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorSnapshottingTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorSnapshottingTest.java @@ -32,6 +32,7 @@ import org.eclipse.ditto.base.model.signals.events.AbstractEventsourcedEvent; import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; import org.eclipse.ditto.internal.utils.persistence.mongo.DittoBsonJson; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.internal.utils.test.Retry; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; @@ -415,7 +416,8 @@ private static void assertPolicyInJournal(final Policy actualPolicy, final Polic } private ActorRef createPersistenceActorFor(final PolicyId policyId) { - final Props props = PolicyPersistenceActor.propsForTests(policyId, pubSubMediator, actorSystem.deadLetters(), + final Props props = PolicyPersistenceActor.propsForTests(policyId, Mockito.mock(MongoReadJournal.class), + pubSubMediator, actorSystem.deadLetters(), actorSystem); return actorSystem.actorOf(props); } diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorTest.java index 655ef28f243..01a541a78f8 100755 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorTest.java @@ -48,6 +48,7 @@ import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.internal.utils.cluster.ShardRegionExtractor; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceActor; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; @@ -1389,7 +1390,8 @@ public void ensureExpiredSubjectIsRemovedDuringRecovery() throws InterruptedExce ClusterShardingSettings.apply(actorSystem).withRole("policies"); final var box = new AtomicReference(); final ActorRef announcementManager = createAnnouncementManager(policyId, box::get); - final Props props = PolicyPersistenceActor.propsForTests(policyId, pubSubMediator, announcementManager, + final Props props = PolicyPersistenceActor.propsForTests(policyId, Mockito.mock(MongoReadJournal.class), + pubSubMediator, announcementManager, actorSystem); final Cluster cluster = Cluster.get(actorSystem); cluster.join(cluster.selfAddress()); @@ -1640,7 +1642,8 @@ public void checkForActivityOfNonexistentPolicy() { final var box = new AtomicReference(); final ActorRef announcementManager = createAnnouncementManager(policyId, box::get); final Props persistentActorProps = - PolicyPersistenceActor.propsForTests(policyId, pubSubMediator, announcementManager, actorSystem); + PolicyPersistenceActor.propsForTests(policyId, Mockito.mock(MongoReadJournal.class), + pubSubMediator, announcementManager, actorSystem); final TestProbe errorsProbe = TestProbe.apply(actorSystem); @@ -1755,7 +1758,8 @@ private ActorRef createPersistenceActorFor(final TestKit testKit, final Policy p private ActorRef createPersistenceActorFor(final TestKit testKit, final PolicyId policyId) { final var box = new AtomicReference(); final ActorRef announcementManager = createAnnouncementManager(policyId, box::get); - final Props props = PolicyPersistenceActor.propsForTests(policyId, pubSubMediator, announcementManager, + final Props props = PolicyPersistenceActor.propsForTests(policyId, Mockito.mock(MongoReadJournal.class), + pubSubMediator, announcementManager, actorSystem); final var persistenceActor = testKit.watch(testKit.childActorOf(props)); box.set(persistenceActor); diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceOperationsActorIT.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceOperationsActorIT.java index 542a6ab07ea..8ee2759eec5 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceOperationsActorIT.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceOperationsActorIT.java @@ -28,6 +28,7 @@ import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; import org.eclipse.ditto.internal.utils.persistence.mongo.MongoClientWrapper; import org.eclipse.ditto.internal.utils.persistence.mongo.ops.eventsource.MongoEventSourceITAssertions; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistence.operations.EntityPersistenceOperations; import org.eclipse.ditto.internal.utils.persistence.operations.NamespacePersistenceOperations; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; @@ -236,9 +237,8 @@ private ActorRef startActorUnderTest(final ActorSystem actorSystem, final ActorR @Override protected ActorRef startEntityActor(final ActorSystem system, final ActorRef pubSubMediator, final PolicyId id) { final Props props = - PolicySupervisorActor.props(pubSubMediator, Mockito.mock(DistributedPub.class), null, Mockito.mock( - PolicyEnforcerProvider.class)); - + PolicySupervisorActor.props(pubSubMediator, Mockito.mock(DistributedPub.class), null, + Mockito.mock(PolicyEnforcerProvider.class), Mockito.mock(MongoReadJournal.class)); return system.actorOf(props, id.toString()); } diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActorTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActorTest.java index 252bf13c47c..12f659dcd31 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActorTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActorTest.java @@ -20,6 +20,7 @@ import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.internal.utils.cluster.StopShardedActor; import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.policies.enforcement.PolicyEnforcer; @@ -35,6 +36,7 @@ import org.junit.ClassRule; import org.junit.Test; import org.mockito.Mock; +import org.mockito.Mockito; import akka.stream.Attributes; import akka.testkit.TestProbe; @@ -80,7 +82,8 @@ public void setup() { public void stopNonexistentPolicy() { new TestKit(actorSystem) {{ final PolicyId policyId = PolicyId.of("test.ns", "stopNonexistentPolicy"); - final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, policyEnforcerProvider); + final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, + policyEnforcerProvider, Mockito.mock(MongoReadJournal.class)); final var underTest = watch(childActorOf(props, policyId.toString())); underTest.tell(new StopShardedActor(), getRef()); expectTerminated(underTest); @@ -91,7 +94,8 @@ public void stopNonexistentPolicy() { public void stopAfterRetrievingNonexistentPolicy() { new TestKit(actorSystem) {{ final PolicyId policyId = PolicyId.of("test.ns", "retrieveNonexistentPolicy"); - final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, policyEnforcerProvider); + final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, + policyEnforcerProvider, Mockito.mock(MongoReadJournal.class)); final var underTest = watch(childActorOf(props, policyId.toString())); final var probe = TestProbe.apply(actorSystem); final var retrievePolicy = RetrievePolicy.of(policyId, DittoHeaders.empty()); @@ -111,7 +115,8 @@ public void stopAfterRetrievingExistingPolicy() { { final var policy = createPolicyWithRandomId(); final var policyId = policy.getEntityId().orElseThrow(); - final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, policyEnforcerProvider); + final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, + policyEnforcerProvider, Mockito.mock(MongoReadJournal.class)); final var underTest = watch(childActorOf(props, policyId.toString())); final var probe = TestProbe.apply(actorSystem); diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/AbstractPolicyCommandStrategyTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/AbstractPolicyCommandStrategyTest.java index 11376735046..38cd8480b17 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/AbstractPolicyCommandStrategyTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/AbstractPolicyCommandStrategyTest.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletionStage; import java.util.function.Consumer; import javax.annotation.Nullable; @@ -111,7 +112,8 @@ void assertModificationResult( final Dummy mock = Dummy.mock(); result.accept(cast(mock)); - verify(mock).onMutation(any(), event.capture(), response.capture(), anyBoolean(), eq(false)); + verify(mock).onMutation(any(), event.capture(), response.capture(), + anyBoolean(), eq(false)); assertThat(event.getValue()).isInstanceOf(expectedEventClass); assertThat(response.getValue()).isInstanceOf(expectedResponseClass); @@ -119,8 +121,8 @@ void assertModificationResult( .describedAs("Event satisfactions failed for expected event of type '%s'", expectedEventClass.getSimpleName()) .satisfies(eventSatisfactions); - assertThat(response.getValue()) - .describedAs("Response predicate failed for expected response of type '%s'", + assertThat((R) response.getValue()) + .describedAs("Response predicate failed for expected responseStage of type '%s'", expectedResponseClass.getSimpleName()) .satisfies(responseSatisfactions); } @@ -191,18 +193,33 @@ static > E getEvent(final Result result) { final List box = new ArrayList<>(1); result.accept(new ResultVisitor<>() { @Override - public void onMutation(final Command command, final E event, final WithDittoHeaders response, + public void onMutation(final Command command, final E event, + final WithDittoHeaders response, final boolean becomeCreated, final boolean becomeDeleted) { box.add(event); } + @Override + public void onStagedMutation(final Command command, final CompletionStage event, + final CompletionStage response, + final boolean becomeCreated, + final boolean becomeDeleted) { + + box.add(event.toCompletableFuture().join()); + } + @Override public void onQuery(final Command command, final WithDittoHeaders response) { throw new AssertionError("Expect mutation result, got query response: " + response); } + @Override + public void onStagedQuery(final Command command, final CompletionStage response) { + throw new AssertionError("Expect mutation result, got query response: " + response); + } + @Override public void onError(final DittoRuntimeException error, final Command errorCausingCommand) { throw new AssertionError("Expect mutation result, got error: " + error); diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PoliciesConditionalHeadersValidatorTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PoliciesConditionalHeadersValidatorTest.java index e51c2af81d2..cbf4820d0df 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PoliciesConditionalHeadersValidatorTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PoliciesConditionalHeadersValidatorTest.java @@ -15,6 +15,7 @@ import static java.text.MessageFormat.format; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.assertThatNoException; import static org.eclipse.ditto.base.model.signals.commands.Command.Category.DELETE; import static org.eclipse.ditto.base.model.signals.commands.Command.Category.MODIFY; import static org.eclipse.ditto.base.model.signals.commands.Command.Category.QUERY; @@ -23,6 +24,7 @@ import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; +import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -30,14 +32,21 @@ import org.assertj.core.api.ThrowableAssertAlternative; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.IfEqual; import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.base.model.headers.entitytag.EntityTagMatchers; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.Command.Category; import org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.policies.model.EffectedImports; +import org.eclipse.ditto.policies.model.Label; +import org.eclipse.ditto.policies.model.Policy; +import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.policies.model.PolicyImport; import org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyPreconditionFailedException; import org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyPreconditionNotModifiedException; +import org.eclipse.ditto.policies.model.signals.commands.modify.ModifyPolicyImport; import org.junit.Test; /** @@ -130,6 +139,57 @@ public void throwsPolicyPreconditionNotModifiedExceptionWhenIfNoneMatchFailsAndC assertNotModified(ifMatchHeaderValue, ifNoneMatchHeaderValue, actualEntityTag, expectedMessage, null); } + @Test + public void ifEqualDoesThrowExceptionWhenIfEqualSkipAndValueIsEqual() { + final PolicyId policyId = PolicyId.generateRandom(); + final PolicyImport policyImport = PolicyImport.newInstance(PolicyId.of("some.policy:one"), EffectedImports.newInstance( + List.of(Label.of("SOME")))); + final Policy policy = Policy.newBuilder() + .setId(policyId) + .setPolicyImport(policyImport) + .build(); + final ModifyPolicyImport command = ModifyPolicyImport.of(policyId, policyImport, + DittoHeaders.newBuilder().ifEqual(IfEqual.SKIP).build()); + + assertThatExceptionOfType(PolicyPreconditionNotModifiedException.class) + .isThrownBy(() -> SUT.applyIfEqualHeader(command, policy)) + .withMessage("The previous value was equal to the new value and the 'if-equal' header was set to 'skip'."); + } + + @Test + public void ifEqualDoesNotThrowExceptionWhenIfEqualSkipAndValueIsNotEqual() { + final PolicyId policyId = PolicyId.generateRandom(); + final PolicyImport policyImport = PolicyImport.newInstance(PolicyId.of("some.policy:one"), EffectedImports.newInstance( + List.of(Label.of("SOME")))); + final Policy policy = Policy.newBuilder() + .setId(policyId) + .setPolicyImport(policyImport) + .build(); + final ModifyPolicyImport command = ModifyPolicyImport.of(policyId, PolicyImport.newInstance(PolicyId.of("some.policy:one"), EffectedImports.newInstance( + List.of(Label.of("OTHER")))), + DittoHeaders.newBuilder().ifEqual(IfEqual.SKIP).build()); + + assertThatNoException() + .isThrownBy(() -> SUT.applyIfEqualHeader(command, policy)); + } + + @Test + public void ifEqualDoesNotThrowExceptionWhenIfEqualUpdateAndValueIsEqual() { + final PolicyId policyId = PolicyId.generateRandom(); + final PolicyImport policyImport = PolicyImport.newInstance(PolicyId.of("some.policy:one"), EffectedImports.newInstance( + List.of(Label.of("SOME")))); + final Policy policy = Policy.newBuilder() + .setId(policyId) + .setPolicyImport(policyImport) + .build(); + final ModifyPolicyImport command = ModifyPolicyImport.of(policyId, policyImport, + DittoHeaders.newBuilder().ifEqual(IfEqual.UPDATE).build()); + + assertThatNoException() + .isThrownBy(() -> SUT.applyIfEqualHeader(command, policy)); + } + + private Command createCommandMock(final Category commandCategory, final String ifMatchHeaderValue, final String ifNoneMatchHeaderValue, final @Nullable JsonObject selectedFields) { final DittoHeaders dittoHeaders = DittoHeaders.newBuilder() diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PolicyConflictStrategyTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PolicyConflictStrategyTest.java index 42c38c7e741..71ea8980047 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PolicyConflictStrategyTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/commands/PolicyConflictStrategyTest.java @@ -18,6 +18,8 @@ import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; +import java.util.concurrent.CompletionStage; + import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; @@ -100,13 +102,26 @@ private ExpectErrorVisitor(final Class clazz) { } @Override - public void onMutation(final Command command, final PolicyEvent event, final WithDittoHeaders response, - final boolean becomeCreated, final boolean becomeDeleted) { + public void onMutation(final Command command, final PolicyEvent event, + final WithDittoHeaders response, final boolean becomeCreated, + final boolean becomeDeleted) { + throw new AssertionError("Expect error, got mutation: " + event); + } + + @Override + public void onStagedMutation(final Command command, final CompletionStage> event, + final CompletionStage response, final boolean becomeCreated, + final boolean becomeDeleted) { throw new AssertionError("Expect error, got mutation: " + event); } @Override - public void onQuery(final Command command, final WithDittoHeaders response) { + public void onQuery(final Command command, final WithDittoHeaders response) { + throw new AssertionError("Expect error, got query: " + response); + } + + @Override + public void onStagedQuery(final Command command, final CompletionStage response) { throw new AssertionError("Expect error, got query: " + response); } diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalCommandRegistryTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalCommandRegistryTest.java index 3360e4699d1..e2c9f1beb87 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalCommandRegistryTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalCommandRegistryTest.java @@ -17,6 +17,7 @@ import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; import org.eclipse.ditto.internal.models.streaming.SudoStreamPids; import org.eclipse.ditto.internal.utils.health.RetrieveHealth; @@ -47,7 +48,8 @@ public PoliciesServiceGlobalCommandRegistryTest() { PurgeEntities.class, PublishSignal.class, ModifyPolicyImports.class, - ModifySplitBrainResolver.class + ModifySplitBrainResolver.class, + SubscribeForPersistedEvents.class ); } } diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalEventRegistryTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalEventRegistryTest.java index a1bcd6a5385..2d73db04b1d 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalEventRegistryTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalEventRegistryTest.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.policies.service.starter; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; import org.eclipse.ditto.internal.utils.persistentactors.EmptyEvent; import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases; import org.eclipse.ditto.policies.model.signals.events.ResourceDeleted; @@ -21,7 +22,8 @@ public final class PoliciesServiceGlobalEventRegistryTest extends GlobalEventReg public PoliciesServiceGlobalEventRegistryTest() { super( ResourceDeleted.class, - EmptyEvent.class + EmptyEvent.class, + StreamingSubscriptionComplete.class ); } diff --git a/pom.xml b/pom.xml index 03e36f5c9cb..3c1fbbbaf9d 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ Eclipse Ditto Eclipse Ditto is a framework for creating and managing digital twins in the IoT. - https://eclipse.org/ditto/ + https://eclipse.dev/ditto/ 2017 @@ -39,7 +39,7 @@ GitHub Issues - https://github.com/eclipse/ditto/issues + https://github.com/eclipse-ditto/ditto/issues @@ -86,10 +86,10 @@ tjaeckle Thomas Jaeckle - thomas.jaeckle@bosch.io + thomas.jaeckle@beyonnex.io https://github.com/thjaeckle - Bosch.IO GmbH - https://www.bosch.io + beyonnex.io GmbH + https://beyonnex.io Lead Committer @@ -132,10 +132,8 @@ jfickel Juergen Fickel - juergen.fickel@bosch.io + eclipse-foundation@retujo.de https://github.com/jufickel-b - Bosch.IO GmbH - https://www.bosch.io Committer @@ -233,16 +231,16 @@ yyyy-MM-dd ${maven.build.timestamp} - scm:git:git@github.com:eclipse/ditto.git - scm:git:https://github.com/eclipse/ditto.git - https://github.com/eclipse/ditto.git + scm:git:git@github.com:eclipse-ditto/ditto.git + scm:git:https://github.com/eclipse-ditto/ditto.git + https://github.com/eclipse-ditto/ditto.git -Dfile.encoding=${project.build.sourceEncoding} - 3.1.0 + 3.3.0 1.7 1.8 @@ -457,7 +455,7 @@ Eclipse Ditto EPL-2.0 - https://eclipse.org/ditto/ + https://eclipse.dev/ditto/ <_noee>true JavaSE-1.8 diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableMessagePath.java b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableMessagePath.java index cb30d319146..6c932733629 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableMessagePath.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableMessagePath.java @@ -71,6 +71,27 @@ public Optional getDirection() { .flatMap(MessagePath::jsonKeyToDirection)); } + @Override + public Optional getMessageSubject() { + if (isInboxOutboxMessage()) { + return Optional.ofNullable( + jsonPointer.getRoot() + .flatMap(MessagePath::jsonKeyToDirection) + .flatMap(direction -> jsonPointer.getSubPointer(2)) + .orElseGet(() -> jsonPointer.getRoot() + .filter(FEATURES::equals) + .flatMap(features -> jsonPointer.get(2)) + .flatMap(MessagePath::jsonKeyToDirection) + .flatMap(direction -> jsonPointer.getSubPointer(4)) + .orElse(null) + ) + ).map(JsonPointer::toString) + .map(s -> s.startsWith("/") ? s.substring(1) : s); + } else { + return Optional.empty(); + } + } + @Override public JsonPointer addLeaf(final JsonKey key) { return jsonPointer.addLeaf(key); diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutablePayload.java b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutablePayload.java index 291c49ca3e3..fa41e6de4f7 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutablePayload.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutablePayload.java @@ -244,7 +244,7 @@ public PayloadBuilder withPath(@Nullable final JsonPointer path) { } @Override - public ImmutablePayloadBuilder withValue(final JsonValue value) { + public ImmutablePayloadBuilder withValue(@Nullable final JsonValue value) { this.value = value; return this; } @@ -280,14 +280,14 @@ public ImmutablePayloadBuilder withMetadata(@Nullable final Metadata metadata) { } @Override - public ImmutablePayloadBuilder withFields(final JsonFieldSelector fields) { + public ImmutablePayloadBuilder withFields(@Nullable final JsonFieldSelector fields) { this.fields = fields; return this; } @Override - public ImmutablePayloadBuilder withFields(final String fields) { - this.fields = JsonFieldSelector.newInstance(fields); + public ImmutablePayloadBuilder withFields(@Nullable final String fields) { + this.fields = null != fields ? JsonFieldSelector.newInstance(fields) : null; return this; } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableTopicPath.java b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableTopicPath.java index 65b193e07e6..8bcc89c638d 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableTopicPath.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableTopicPath.java @@ -30,6 +30,7 @@ import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.NotThreadSafe; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonKey; import org.eclipse.ditto.json.JsonPointer; @@ -48,6 +49,7 @@ final class ImmutableTopicPath implements TopicPath { private final Criterion criterion; @Nullable private final Action action; @Nullable private final SearchAction searchAction; + @Nullable private final StreamingAction streamingAction; @Nullable private final String subject; private ImmutableTopicPath(final Builder builder) { @@ -58,6 +60,7 @@ private ImmutableTopicPath(final Builder builder) { criterion = builder.criterion; action = builder.action; searchAction = builder.searchAction; + streamingAction = builder.streamingAction; subject = builder.subject; } @@ -139,6 +142,11 @@ public Optional getSearchAction() { return Optional.ofNullable(searchAction); } + @Override + public Optional getStreamingAction() { + return Optional.ofNullable(streamingAction); + } + @Override public Optional getSubject() { return Optional.ofNullable(subject); @@ -159,6 +167,7 @@ public String getPath() { .add(criterion.getName()) .add(getStringOrNull(action)) .add(getStringOrNull(searchAction)) + .add(getStringOrNull(streamingAction)) .add(getStringOrNull(subject)) .build(); return pathPartStream.filter(Objects::nonNull).collect(Collectors.joining(PATH_DELIMITER)); @@ -212,12 +221,14 @@ public boolean equals(final Object o) { criterion == that.criterion && Objects.equals(action, that.action) && Objects.equals(searchAction, that.searchAction) && + Objects.equals(streamingAction, that.streamingAction) && Objects.equals(subject, that.subject); } @Override public int hashCode() { - return Objects.hash(namespace, name, group, channel, criterion, action, searchAction, subject); + return Objects.hash(namespace, name, group, channel, criterion, action, searchAction, streamingAction, + subject); } @Override @@ -230,6 +241,7 @@ public String toString() { ", criterion=" + criterion + ", action=" + action + ", searchAction=" + searchAction + + ", streamingAction=" + streamingAction + ", subject=" + subject + ", path=" + getPath() + "]"; @@ -241,7 +253,8 @@ public String toString() { @NotThreadSafe private static final class Builder implements TopicPathBuilder, MessagesTopicPathBuilder, EventsTopicPathBuilder, CommandsTopicPathBuilder, - AcknowledgementTopicPathBuilder, SearchTopicPathBuilder, AnnouncementsTopicPathBuilder { + AcknowledgementTopicPathBuilder, SearchTopicPathBuilder, AnnouncementsTopicPathBuilder, + StreamingTopicPathBuilder { private final String namespace; private final String name; @@ -251,6 +264,7 @@ private static final class Builder private Criterion criterion; @Nullable private Action action; @Nullable private SearchAction searchAction; + @Nullable private StreamingAction streamingAction; @Nullable private String subject; private Builder(final String namespace, final String name) { @@ -261,6 +275,7 @@ private Builder(final String namespace, final String name) { criterion = null; action = null; searchAction = null; + streamingAction = null; subject = null; } @@ -318,6 +333,12 @@ public AnnouncementsTopicPathBuilder announcements() { return this; } + @Override + public StreamingTopicPathBuilder streaming() { + criterion = Criterion.STREAMING; + return this; + } + @Override public EventsTopicPathBuilder events() { criterion = Criterion.EVENTS; @@ -378,45 +399,83 @@ public TopicPathBuildable subscribe() { return this; } + @Override + public TopicPathBuildable subscribe(final String subscribingCommandName) { + if (subscribingCommandName.equals(SubscribeForPersistedEvents.NAME)) { + streamingAction = StreamingAction.SUBSCRIBE_FOR_PERSISTED_EVENTS; + } else { + throw UnknownCommandException.newBuilder(subscribingCommandName).build(); + } + return this; + } + @Override public TopicPathBuildable cancel() { - searchAction = SearchAction.CANCEL; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.CANCEL; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.CANCEL; + } return this; } @Override public TopicPathBuildable request() { - searchAction = SearchAction.REQUEST; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.REQUEST; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.REQUEST; + } return this; } @Override public TopicPathBuildable complete() { - searchAction = SearchAction.COMPLETE; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.COMPLETE; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.COMPLETE; + } return this; } @Override public TopicPathBuildable failed() { - searchAction = SearchAction.FAILED; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.FAILED; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.FAILED; + } return this; } @Override public TopicPathBuildable hasNext() { - searchAction = SearchAction.NEXT; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.NEXT; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.NEXT; + } return this; } @Override public EventsTopicPathBuilder generated() { - searchAction = SearchAction.GENERATED; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.GENERATED; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.GENERATED; + } return this; } @Override public TopicPathBuildable error() { - searchAction = SearchAction.ERROR; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.ERROR; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.ERROR; + } return this; } @@ -521,6 +580,9 @@ public ImmutableTopicPath get() { case SEARCH: topicPathBuilder.searchAction = tryToGetSearchActionForName(tryToGetSearchActionName()); break; + case STREAMING: + topicPathBuilder.streamingAction = tryToGetStreamingActionForName(tryToGetSearchActionName()); + break; case ERRORS: break; case MESSAGES: @@ -656,6 +718,13 @@ private SearchAction tryToGetSearchActionForName(final String searchActionName) .build()); } + private StreamingAction tryToGetStreamingActionForName(final String streamingActionName) { + return StreamingAction.forName(streamingActionName) + .orElseThrow(() -> UnknownTopicPathException.newBuilder(topicPathString) + .description(MessageFormat.format("Streaming action name <{0}> is unknown.", streamingActionName)) + .build()); + } + @Nullable private String getSubjectOrNull() { final String subject = String.join(TopicPath.PATH_DELIMITER, topicPathParts); diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/MessagePath.java b/protocol/src/main/java/org/eclipse/ditto/protocol/MessagePath.java index 62d7ee446cf..32d4888f0de 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/MessagePath.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/MessagePath.java @@ -37,6 +37,25 @@ public interface MessagePath extends JsonPointer { */ Optional getDirection(); + /** + * Retrieves the "Message" subject in case the path is a message FROM/TO a thing + * (meaning that also {@link #getDirection()} is present). + * + * @return the "Message" subject in case the path is a message FROM/TO a thing. + * @since 3.3.0 + */ + Optional getMessageSubject(); + + /** + * Determines whether this instance represents a path for a Ditto inbox/outbox "message" or not. + * + * @return whether this instance represents a path for a Ditto inbox/outbox "message" or not. + * @since 3.3.0 + */ + default boolean isInboxOutboxMessage() { + return getDirection().isPresent(); + } + static Optional jsonKeyToDirection(final JsonKey jsonKey) { switch (jsonKey.toString()) { case "inbox": diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/ProtocolFactory.java b/protocol/src/main/java/org/eclipse/ditto/protocol/ProtocolFactory.java index 3e4244a46be..021c0a0c142 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/ProtocolFactory.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/ProtocolFactory.java @@ -19,9 +19,11 @@ import java.util.stream.Collectors; import org.eclipse.ditto.base.model.common.DittoConstants; +import org.eclipse.ditto.base.model.entity.id.EntityId; import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.contenttype.ContentType; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonPointer; @@ -126,6 +128,38 @@ public static TopicPathBuilder newTopicPathBuilder(final NamespacedEntityId enti return result; } + /** + * Returns a new {@code TopicPathBuilder} for the specified {@link EntityId}. + * The namespace and name part of the {@code TopicPath} will pe parsed from the entity ID and set in the builder. + * + * @param entityId the ID. + * @return the builder. + * @throws NullPointerException if {@code entityId} is {@code null}. + * @throws org.eclipse.ditto.things.model.ThingIdInvalidException if {@code entityId} is not in the expected + * format. + * @since 3.2.0 + */ + public static TopicPathBuilder newTopicPathBuilder(final EntityId entityId) { + checkNotNull(entityId, "entityId"); + final TopicPathBuilder result; + if (entityId instanceof NamespacedEntityId) { + final String namespace = ((NamespacedEntityId) entityId).getNamespace(); + final String name = ((NamespacedEntityId) entityId).getName(); + result = ImmutableTopicPath.newBuilder(namespace, name); + } else { + result = ProtocolFactory.newTopicPathBuilderFromName(entityId.toString()); + } + + if (entityId.getEntityType().equals(ThingConstants.ENTITY_TYPE)) { + return result.things(); + } else if (entityId.getEntityType().equals(PolicyConstants.ENTITY_TYPE)) { + return result.policies(); + } else if (entityId.getEntityType().equals(ConnectivityConstants.ENTITY_TYPE)) { + return result.connections(); + } + return result; + } + /** * Returns a new {@code TopicPathBuilder} for the specified {@link PolicyId}. * The namespace and name part of the {@code TopicPath} will pe parsed from the {@code PolicyId} and set in the diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/StreamingTopicPathBuilder.java b/protocol/src/main/java/org/eclipse/ditto/protocol/StreamingTopicPathBuilder.java new file mode 100644 index 00000000000..5c1af097bbd --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/StreamingTopicPathBuilder.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol; + +/** + * Builder to create a topic path for streaming commands. + * + * @since 3.2.0 + */ +public interface StreamingTopicPathBuilder extends TopicPathBuildable { + + /** + * Sets the {@code Action} of this builder to the passed {@code subscribingCommandName}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable subscribe(String subscribingCommandName); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#CANCEL}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable cancel(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#REQUEST}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable request(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#COMPLETE}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable complete(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#FAILED}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable failed(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#NEXT}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable hasNext(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#GENERATED}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable generated(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#ERROR}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable error(); + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPath.java b/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPath.java index 3af44dd32af..6fa1c1f0906 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPath.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPath.java @@ -19,6 +19,14 @@ import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionCreated; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionHasNext; +import org.eclipse.ditto.connectivity.model.ConnectionId; import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.policies.model.PolicyConstants; import org.eclipse.ditto.policies.model.PolicyId; @@ -68,6 +76,18 @@ static TopicPathBuilder newBuilder(final PolicyId policyId) { return ProtocolFactory.newTopicPathBuilder(policyId); } + /** + * Returns a mutable builder to create immutable {@code TopicPath} instances for a given {@code connectionId}. + * + * @param connectionId the identifier of the {@code Connection}. + * @return the builder. + * @throws NullPointerException if {@code connectionId} is {@code null}. + * @since 3.1.0 + */ + static TopicPathBuilder newBuilder(final ConnectionId connectionId) { + return ProtocolFactory.newTopicPathBuilder(connectionId); + } + /** * Returns a mutable builder to create immutable {@code TopicPath} instances for a given {@code namespace}. * @@ -122,12 +142,20 @@ static TopicPathBuilder fromNamespace(final String namespace) { Optional getAction(); /** - * Returns an {@link Optional} for an search action part of this {@code TopicPath}. + * Returns an {@link Optional} for a search action part of this {@code TopicPath}. * * @return the search action. */ Optional getSearchAction(); + /** + * Returns an {@link Optional} for a streaming action part of this {@code TopicPath}. + * + * @return the streaming action. + * @since 3.2.0 + */ + Optional getStreamingAction(); + /** * Returns an {@link Optional} for a subject part of this {@code TopicPath}. * @@ -268,7 +296,14 @@ enum Criterion { * * @since 2.0.0 */ - ANNOUNCEMENTS("announcements"); + ANNOUNCEMENTS("announcements"), + + /** + * Criterion for streaming commands. + * + * @since 3.2.0 + */ + STREAMING("streaming"); private final String name; @@ -482,4 +517,65 @@ public String toString() { } + /** + * An enumeration of topic path streaming actions. + * + * @since 3.2.0 + */ + enum StreamingAction { + + SUBSCRIBE_FOR_PERSISTED_EVENTS(SubscribeForPersistedEvents.NAME), + + CANCEL(CancelStreamingSubscription.NAME), + + REQUEST(RequestFromStreamingSubscription.NAME), + + COMPLETE(StreamingSubscriptionComplete.NAME), + + GENERATED(StreamingSubscriptionCreated.NAME), + + FAILED(StreamingSubscriptionFailed.NAME), + + NEXT(StreamingSubscriptionHasNext.NAME), + + ERROR("error"); + + private final String name; + + StreamingAction(final String name) { + this.name = name; + } + + /** + * Creates a StreamingAction from the passed StreamingAction {@code name} if such an enum value exists, + * otherwise an empty Optional. + * + * @param name the StreamingAction name to create the StreamingAction enum value of. + * @return the optional StreamingAction. + */ + public static Optional forName(final String name) { + return Stream.of(values()) + .filter(a -> Objects.equals(a.getName(), name)) + .findFirst(); + } + + /** + * Returns the StreamingAction name as String. + * + * @return the StreamingAction name as String. + */ + public String getName() { + return name; + } + + /** + * @return the same as {@link #getName()}. + */ + @Override + public String toString() { + return getName(); + } + + } + } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPathBuilder.java b/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPathBuilder.java index 6c4f9db8e5e..31532183d0a 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPathBuilder.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPathBuilder.java @@ -64,6 +64,15 @@ public interface TopicPathBuilder { */ AnnouncementsTopicPathBuilder announcements(); + /** + * Sets the {@code Group} of this builder to {@link TopicPath.Criterion#STREAMING}. A previously set group is + * replaced. + * + * @return this builder. + * @since 3.2.0 + */ + StreamingTopicPathBuilder streaming(); + /** * Sets the {@code Channel} of this builder to {@link TopicPath.Channel#TWIN}. A previously set channel is * replaced. diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AbstractStreamingMessageAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AbstractStreamingMessageAdapter.java new file mode 100644 index 00000000000..161eb40311d --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AbstractStreamingMessageAdapter.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.adapter; + +import java.util.EnumSet; +import java.util.Set; + +import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.mapper.SignalMapper; +import org.eclipse.ditto.protocol.mappingstrategies.MappingStrategies; + +/** + * Adapter for mapping a "streaming" {@link Signal} to and from an {@link Adaptable}. + * + * @param the type of the signals mapped by this adapter. + */ +abstract class AbstractStreamingMessageAdapter> extends AbstractAdapter + implements Adapter { + + private final SignalMapper signalMapper; + + AbstractStreamingMessageAdapter( + final MappingStrategies mappingStrategies, + final SignalMapper signalMapper, + final HeaderTranslator headerTranslator) { + + super(mappingStrategies, headerTranslator, EmptyPathMatcher.getInstance()); + this.signalMapper = signalMapper; + } + + @Override + protected Adaptable mapSignalToAdaptable(final T signal, final TopicPath.Channel channel) { + return signalMapper.mapSignalToAdaptable(signal, channel); + } + + @Override + public Adaptable toAdaptable(final T t) { + return toAdaptable(t, TopicPath.Channel.LIVE); + } + + @Override + public TopicPath toTopicPath(final T signal, final TopicPath.Channel channel) { + return signalMapper.mapSignalToTopicPath(signal, channel); + } + + @Override + public Set getGroups() { + return EnumSet.of(TopicPath.Group.POLICIES, TopicPath.Group.THINGS, TopicPath.Group.CONNECTIONS); + } + + @Override + public Set getChannels() { + return EnumSet.of(TopicPath.Channel.NONE, TopicPath.Channel.TWIN); + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/Adapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/Adapter.java index 8a8ec6b22b8..4bd0a3a024e 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/Adapter.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/Adapter.java @@ -113,6 +113,17 @@ default Set getSearchActions() { return Collections.emptySet(); } + /** + * Return the set of streaming actions supported by this adapter. + * It is the empty set by default. + * + * @return the collection of supported streaming actions. + * @since 3.2.0 + */ + default Set getStreamingActions() { + return Collections.emptySet(); + } + /** * Retrieve whether this adapter is for responses. * diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AdapterResolverBySignal.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AdapterResolverBySignal.java index ed3bdfdb389..6e8ca806a45 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AdapterResolverBySignal.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AdapterResolverBySignal.java @@ -23,7 +23,9 @@ import org.eclipse.ditto.base.model.signals.acks.Acknowledgements; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.connectivity.model.signals.announcements.ConnectivityAnnouncement; import org.eclipse.ditto.messages.model.signals.commands.MessageCommand; import org.eclipse.ditto.messages.model.signals.commands.MessageCommandResponse; @@ -33,6 +35,7 @@ import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommand; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.protocol.TopicPath; import org.eclipse.ditto.protocol.UnknownChannelException; import org.eclipse.ditto.protocol.UnknownSignalException; @@ -61,16 +64,22 @@ final class AdapterResolverBySignal { private final PolicyCommandAdapterProvider policiesAdapters; private final ConnectivityCommandAdapterProvider connectivityAdapters; private final AcknowledgementAdapterProvider acknowledgementAdapters; + private final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter; + private final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter; AdapterResolverBySignal(final ThingCommandAdapterProvider thingsAdapters, final PolicyCommandAdapterProvider policiesAdapters, final ConnectivityCommandAdapterProvider connectivityAdapters, - final AcknowledgementAdapterProvider acknowledgementAdapters) { + final AcknowledgementAdapterProvider acknowledgementAdapters, + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter, + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter) { this.thingsAdapters = thingsAdapters; this.policiesAdapters = policiesAdapters; this.connectivityAdapters = connectivityAdapters; this.acknowledgementAdapters = acknowledgementAdapters; + this.streamingSubscriptionCommandAdapter = streamingSubscriptionCommandAdapter; + this.streamingSubscriptionEventAdapter = streamingSubscriptionEventAdapter; } @SuppressWarnings("unchecked") @@ -110,10 +119,19 @@ private > Adapter resolveEvent(final Event event, fina validateChannel(channel, event, LIVE, TWIN); return (Adapter) thingsAdapters.getEventAdapter(); } + if (event instanceof PolicyEvent) { + validateChannel(channel, event, NONE); + return (Adapter) policiesAdapters.getEventAdapter(); + } + if (event instanceof SubscriptionEvent) { validateNotLive(event); return (Adapter) thingsAdapters.getSubscriptionEventAdapter(); } + if (event instanceof StreamingSubscriptionEvent) { + validateNotLive(event); + return (Adapter) streamingSubscriptionEventAdapter; + } throw UnknownSignalException.newBuilder(event.getName()) .dittoHeaders(event.getDittoHeaders()) @@ -212,6 +230,10 @@ private > Adapter resolveCommand(final Command command validateNotLive(command); return (Adapter) thingsAdapters.getSearchCommandAdapter(); } + if (command instanceof StreamingSubscriptionCommand) { + validateNotLive(command); + return (Adapter) streamingSubscriptionCommandAdapter; + } if (command instanceof PolicyModifyCommand) { validateChannel(channel, command, NONE); diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DefaultAdapterResolver.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DefaultAdapterResolver.java index fd7b522b4d0..44717e7fcfb 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DefaultAdapterResolver.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DefaultAdapterResolver.java @@ -43,15 +43,19 @@ final class DefaultAdapterResolver implements AdapterResolver { DefaultAdapterResolver(final ThingCommandAdapterProvider thingsAdapters, final PolicyCommandAdapterProvider policiesAdapters, final ConnectivityCommandAdapterProvider connectivityAdapters, - final AcknowledgementAdapterProvider acknowledgementAdapters) { + final AcknowledgementAdapterProvider acknowledgementAdapters, + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter, + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter) { final List> adapters = new ArrayList<>(); adapters.addAll(thingsAdapters.getAdapters()); adapters.addAll(policiesAdapters.getAdapters()); adapters.addAll(connectivityAdapters.getAdapters()); adapters.addAll(acknowledgementAdapters.getAdapters()); + adapters.add(streamingSubscriptionCommandAdapter); + adapters.add(streamingSubscriptionEventAdapter); resolver = computeResolver(adapters); resolverBySignal = new AdapterResolverBySignal(thingsAdapters, policiesAdapters, connectivityAdapters, - acknowledgementAdapters); + acknowledgementAdapters, streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter); } @Override @@ -236,6 +240,8 @@ private static Function> computeResolver(final List(TopicPath.SearchAction.class, TopicPath.SearchAction.values(), Adapter::getSearchActions, forTopicPath(TopicPath::getSearchAction)), + new ForEnumOptional<>(TopicPath.StreamingAction.class, TopicPath.StreamingAction.values(), + Adapter::getStreamingActions, forTopicPath(TopicPath::getStreamingAction)), new ForEnum<>(Bool.class, Bool.values(), Bool.composeAsSet(Adapter::isForResponses), Bool.compose(DefaultAdapterResolver::isResponse)), new ForEnum<>(Bool.class, Bool.values(), Bool.composeAsSet(Adapter::requiresSubject), diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapter.java index 3a22c9f4b1f..4f2fa4993bb 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapter.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapter.java @@ -43,6 +43,9 @@ public final class DittoProtocolAdapter implements ProtocolAdapter { private final PolicyCommandAdapterProvider policiesAdapters; private final ConnectivityCommandAdapterProvider connectivityAdapters; private final AcknowledgementAdapterProvider acknowledgementAdapters; + + private final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter; + private final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter; private final AdapterResolver adapterResolver; private DittoProtocolAdapter(final ErrorRegistry errorRegistry, @@ -52,8 +55,10 @@ private DittoProtocolAdapter(final ErrorRegistry errorReg this.policiesAdapters = new DefaultPolicyCommandAdapterProvider(errorRegistry, headerTranslator); this.connectivityAdapters = new DefaultConnectivityCommandAdapterProvider(headerTranslator); this.acknowledgementAdapters = new DefaultAcknowledgementsAdapterProvider(errorRegistry, headerTranslator); + streamingSubscriptionCommandAdapter = StreamingSubscriptionCommandAdapter.of(headerTranslator); + streamingSubscriptionEventAdapter = StreamingSubscriptionEventAdapter.of(headerTranslator, errorRegistry); this.adapterResolver = new DefaultAdapterResolver(thingsAdapters, policiesAdapters, connectivityAdapters, - acknowledgementAdapters); + acknowledgementAdapters, streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter); } private DittoProtocolAdapter(final HeaderTranslator headerTranslator, @@ -61,12 +66,18 @@ private DittoProtocolAdapter(final HeaderTranslator headerTranslator, final PolicyCommandAdapterProvider policiesAdapters, final ConnectivityCommandAdapterProvider connectivityAdapters, final AcknowledgementAdapterProvider acknowledgementAdapters, + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter, + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter, final AdapterResolver adapterResolver) { this.headerTranslator = checkNotNull(headerTranslator, "headerTranslator"); this.thingsAdapters = checkNotNull(thingsAdapters, "thingsAdapters"); this.policiesAdapters = checkNotNull(policiesAdapters, "policiesAdapters"); this.connectivityAdapters = checkNotNull(connectivityAdapters, "connectivityAdapters"); this.acknowledgementAdapters = checkNotNull(acknowledgementAdapters, "acknowledgementAdapters"); + this.streamingSubscriptionCommandAdapter = checkNotNull(streamingSubscriptionCommandAdapter, + "streamingSubscriptionCommandAdapter"); + this.streamingSubscriptionEventAdapter = checkNotNull(streamingSubscriptionEventAdapter, + "streamingSubscriptionEventAdapter"); this.adapterResolver = checkNotNull(adapterResolver, "adapterResolver"); } @@ -106,6 +117,8 @@ public static HeaderTranslator getHeaderTranslator() { * @param policyCommandAdapterProvider command adapters for policy commands * @param connectivityAdapters adapters for connectivity commands. * @param acknowledgementAdapters adapters for acknowledgements. + * @param streamingSubscriptionCommandAdapter adapters for streaming subscription commands. + * @param streamingSubscriptionEventAdapter adapters for streaming subscription events. * @param adapterResolver resolves the correct adapter from a command * @return new instance of {@link DittoProtocolAdapter} */ @@ -114,9 +127,12 @@ static DittoProtocolAdapter newInstance(final HeaderTranslator headerTranslator, final PolicyCommandAdapterProvider policyCommandAdapterProvider, final ConnectivityCommandAdapterProvider connectivityAdapters, final AcknowledgementAdapterProvider acknowledgementAdapters, + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter, + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter, final AdapterResolver adapterResolver) { return new DittoProtocolAdapter(headerTranslator, thingCommandAdapterProvider, policyCommandAdapterProvider, - connectivityAdapters, acknowledgementAdapters, adapterResolver + connectivityAdapters, acknowledgementAdapters, + streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter, adapterResolver ); } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/ProtocolAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/ProtocolAdapter.java index 05cd69b599f..ebd67556eab 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/ProtocolAdapter.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/ProtocolAdapter.java @@ -19,10 +19,14 @@ import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.announcements.Announcement; +import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; +import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommandResponse; +import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.messages.model.signals.commands.MessageCommand; import org.eclipse.ditto.messages.model.signals.commands.MessageCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.PolicyCommand; import org.eclipse.ditto.policies.model.signals.commands.PolicyCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.protocol.Adaptable; import org.eclipse.ditto.protocol.TopicPath; @@ -112,7 +116,11 @@ static TopicPath.Channel determineChannel(final Signal signal) { * @return the default channel determined from the signal */ static TopicPath.Channel determineDefaultChannel(final Signal signal) { - if (signal instanceof PolicyCommand || signal instanceof PolicyCommandResponse) { + if (signal instanceof PolicyCommand || signal instanceof PolicyCommandResponse || + signal instanceof PolicyEvent) { + return NONE; + } else if (signal instanceof ConnectivityCommand || signal instanceof ConnectivityCommandResponse || + signal instanceof ConnectivityEvent) { return NONE; } else if (signal instanceof Announcement) { return NONE; diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionCommandAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionCommandAdapter.java new file mode 100644 index 00000000000..a7a626a6e71 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionCommandAdapter.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.adapter; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.mapper.SignalMapperFactory; +import org.eclipse.ditto.protocol.mappingstrategies.MappingStrategiesFactory; + +/** + * Adapter for mapping a {@link StreamingSubscriptionCommand} to and from an {@link Adaptable}. + * + * @since 3.2.0 + */ +public final class StreamingSubscriptionCommandAdapter + extends AbstractStreamingMessageAdapter> { + + private StreamingSubscriptionCommandAdapter(final HeaderTranslator headerTranslator) { + super(MappingStrategiesFactory.getStreamingSubscriptionCommandMappingStrategies(), + SignalMapperFactory.newStreamingSubscriptionCommandSignalMapper(), + headerTranslator + ); + } + + /** + * Returns a new StreamingSubscriptionCommandAdapter. + * + * @param headerTranslator translator between external and Ditto headers. + * @return the adapter. + */ + public static StreamingSubscriptionCommandAdapter of(final HeaderTranslator headerTranslator) { + return new StreamingSubscriptionCommandAdapter(checkNotNull(headerTranslator, "headerTranslator")); + } + + @Override + protected String getType(final Adaptable adaptable) { + return StreamingSubscriptionCommand.TYPE_PREFIX + adaptable.getTopicPath().getStreamingAction().orElse(null); + } + + @Override + public Set getCriteria() { + return EnumSet.of(TopicPath.Criterion.STREAMING); + } + + @Override + public Set getActions() { + return Collections.emptySet(); + } + + @Override + public boolean isForResponses() { + return false; + } + + @Override + public Set getStreamingActions() { + return EnumSet.of(TopicPath.StreamingAction.SUBSCRIBE_FOR_PERSISTED_EVENTS, TopicPath.StreamingAction.REQUEST, + TopicPath.StreamingAction.CANCEL); + } +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionEventAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionEventAdapter.java new file mode 100644 index 00000000000..f6f22735793 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionEventAdapter.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.adapter; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.base.model.signals.ErrorRegistry; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.mapper.SignalMapperFactory; +import org.eclipse.ditto.protocol.mappingstrategies.MappingStrategiesFactory; + +/** + * Adapter for mapping a {@link StreamingSubscriptionEvent} to and from an {@link Adaptable}. + * + * @since 3.2.0 + */ +public final class StreamingSubscriptionEventAdapter + extends AbstractStreamingMessageAdapter> { + + private StreamingSubscriptionEventAdapter(final HeaderTranslator headerTranslator, + final ErrorRegistry errorRegistry) { + super(MappingStrategiesFactory.getStreamingSubscriptionEventMappingStrategies(errorRegistry), + SignalMapperFactory.newStreamingSubscriptionEventSignalMapper(), + headerTranslator + ); + } + + /** + * Returns a new StreamingSubscriptionEventAdapter. + * + * @param headerTranslator translator between external and Ditto headers. + * @param errorRegistry the error registry for {@code SubscriptionFailed} events. + * @return the adapter. + */ + public static StreamingSubscriptionEventAdapter of(final HeaderTranslator headerTranslator, + final ErrorRegistry errorRegistry) { + return new StreamingSubscriptionEventAdapter(checkNotNull(headerTranslator, "headerTranslator"), + checkNotNull(errorRegistry, "errorRegistry")); + } + + @Override + protected String getType(final Adaptable adaptable) { + return StreamingSubscriptionEvent.TYPE_PREFIX + adaptable.getTopicPath().getStreamingAction().orElse(null); + } + + @Override + public Set getCriteria() { + return EnumSet.of(TopicPath.Criterion.STREAMING); + } + + @Override + public Set getActions() { + return Collections.emptySet(); + } + + @Override + public boolean isForResponses() { + return false; + } + + @Override + public Set getStreamingActions() { + return EnumSet.of(TopicPath.StreamingAction.COMPLETE, TopicPath.StreamingAction.NEXT, + TopicPath.StreamingAction.FAILED, TopicPath.StreamingAction.GENERATED); + } + + @Override + public boolean supportsWildcardTopics() { + return false; + } +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/connectivity/package-info.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/connectivity/package-info.java new file mode 100755 index 00000000000..51b170fb9b5 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/connectivity/package-info.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.eclipse.ditto.utils.jsr305.annotations.AllValuesAreNonnullByDefault +package org.eclipse.ditto.protocol.adapter.connectivity; diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/DefaultPolicyCommandAdapterProvider.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/DefaultPolicyCommandAdapterProvider.java index 37425b6bd07..1fe1bbfd221 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/DefaultPolicyCommandAdapterProvider.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/DefaultPolicyCommandAdapterProvider.java @@ -24,6 +24,7 @@ import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommand; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.protocol.adapter.Adapter; import org.eclipse.ditto.protocol.adapter.provider.PolicyCommandAdapterProvider; @@ -40,6 +41,7 @@ public final class DefaultPolicyCommandAdapterProvider implements PolicyCommandA private final PolicyModifyCommandResponseAdapter policyModifyCommandResponseAdapter; private final PolicyQueryCommandResponseAdapter policyQueryCommandResponseAdapter; private final PolicyAnnouncementAdapter policyAnnouncementAdapter; + private final PolicyEventAdapter policyEventAdapter; public DefaultPolicyCommandAdapterProvider(final ErrorRegistry errorRegistry, final HeaderTranslator headerTranslator) { @@ -49,6 +51,7 @@ public DefaultPolicyCommandAdapterProvider(final ErrorRegistry getErrorResponseAdapter() { @@ -75,6 +78,11 @@ public Adapter> getAnnouncementAdapter() { return policyAnnouncementAdapter; } + @Override + public Adapter> getEventAdapter() { + return policyEventAdapter; + } + @Override public List> getAdapters() { return Arrays.asList( @@ -83,7 +91,8 @@ public List> getAdapters() { policyQueryCommandAdapter, policyModifyCommandResponseAdapter, policyQueryCommandResponseAdapter, - policyAnnouncementAdapter + policyAnnouncementAdapter, + policyEventAdapter ); } } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/PolicyEventAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/PolicyEventAdapter.java new file mode 100644 index 00000000000..589a9a1899c --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/PolicyEventAdapter.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.adapter.policies; + +import static java.util.Objects.requireNonNull; + +import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.adapter.AbstractAdapter; +import org.eclipse.ditto.protocol.adapter.EventAdapter; +import org.eclipse.ditto.protocol.mapper.SignalMapperFactory; +import org.eclipse.ditto.protocol.mappingstrategies.MappingStrategiesFactory; + +/** + * Adapter for mapping a {@link PolicyEvent} to and from an {@link org.eclipse.ditto.protocol.Adaptable}. + */ +final class PolicyEventAdapter extends AbstractPolicyAdapter> implements EventAdapter> { + + private PolicyEventAdapter(final HeaderTranslator headerTranslator) { + super(MappingStrategiesFactory.getPolicyEventMappingStrategies(), + SignalMapperFactory.newPolicyEventSignalMapper(), + headerTranslator); + } + + /** + * Returns a new PolicyEventAdapter. + * + * @param headerTranslator translator between external and Ditto headers. + * @return the adapter. + */ + public static PolicyEventAdapter of(final HeaderTranslator headerTranslator) { + return new PolicyEventAdapter(requireNonNull(headerTranslator)); + } + + private static String getActionNameWithFirstLetterUpperCase(final TopicPath topicPath) { + return topicPath.getAction() + .map(TopicPath.Action::toString) + .map(AbstractAdapter::upperCaseFirst) + .orElseThrow(() -> new NullPointerException("TopicPath did not contain an Action!")); + } + + @Override + protected String getType(final Adaptable adaptable) { + final TopicPath topicPath = adaptable.getTopicPath(); + final JsonPointer path = adaptable.getPayload().getPath(); + final String eventName = payloadPathMatcher.match(path) + getActionNameWithFirstLetterUpperCase(topicPath); + return topicPath.getGroup() + "." + topicPath.getCriterion() + ":" + eventName; + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/provider/PolicyCommandAdapterProvider.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/provider/PolicyCommandAdapterProvider.java index 080b6910fa3..09daf183b45 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/provider/PolicyCommandAdapterProvider.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/provider/PolicyCommandAdapterProvider.java @@ -12,13 +12,14 @@ */ package org.eclipse.ditto.protocol.adapter.provider; -import org.eclipse.ditto.protocol.adapter.Adapter; import org.eclipse.ditto.policies.model.signals.announcements.PolicyAnnouncement; import org.eclipse.ditto.policies.model.signals.commands.PolicyErrorResponse; import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommand; import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommand; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.protocol.adapter.Adapter; /** * Provider for all policy command adapters. This interface mainly defines the generic type arguments and adds @@ -30,6 +31,7 @@ public interface PolicyCommandAdapterProvider extends QueryCommandAdapterProvider, PolicyQueryCommandResponse>, ModifyCommandAdapterProvider, PolicyModifyCommandResponse>, ErrorResponseAdapterProvider, + EventAdapterProvider>, AdapterProvider { /** diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/AbstractCommandSignalMapper.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/AbstractCommandSignalMapper.java index a6fce7e9d97..10182d5902e 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/AbstractCommandSignalMapper.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/AbstractCommandSignalMapper.java @@ -15,12 +15,12 @@ import java.util.stream.Stream; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.protocol.CommandsTopicPathBuilder; import org.eclipse.ditto.protocol.TopicPath; import org.eclipse.ditto.protocol.TopicPathBuilder; import org.eclipse.ditto.protocol.UnknownCommandException; import org.eclipse.ditto.protocol.UnknownCommandResponseException; -import org.eclipse.ditto.base.model.signals.Signal; /** * Base class of {@link SignalMapper}s for commands (e.g. query, modify commands). @@ -45,7 +45,7 @@ TopicPath getTopicPath(final T command, final TopicPath.Channel channel) { abstract TopicPathBuilder getTopicPathBuilder(final T command); /** - * @return array of {@link org.eclipse.ditto.protocol.TopicPath.Action}s the implementation supports. + * @return array of {@link TopicPath.Action}s the implementation supports. */ abstract TopicPath.Action[] getSupportedActions(); diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/PolicyEventSignalMapper.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/PolicyEventSignalMapper.java new file mode 100644 index 00000000000..343d730dd52 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/PolicyEventSignalMapper.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mapper; + +import java.util.Locale; +import java.util.Optional; + +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonValue; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.protocol.EventsTopicPathBuilder; +import org.eclipse.ditto.protocol.PayloadBuilder; +import org.eclipse.ditto.protocol.ProtocolFactory; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.TopicPathBuilder; +import org.eclipse.ditto.protocol.UnknownChannelException; +import org.eclipse.ditto.protocol.UnknownEventException; + +final class PolicyEventSignalMapper extends AbstractSignalMapper> { + + @Override + void enhancePayloadBuilder(final PolicyEvent signal, final PayloadBuilder payloadBuilder) { + payloadBuilder.withRevision(signal.getRevision()) + .withTimestamp(signal.getTimestamp().orElse(null)); + final Optional value = + signal.getEntity(signal.getDittoHeaders().getSchemaVersion().orElse(signal.getLatestSchemaVersion())); + value.ifPresent(payloadBuilder::withValue); + } + + @Override + DittoHeaders enhanceHeaders(final PolicyEvent signal) { + final Optional value = + signal.getEntity(signal.getDittoHeaders().getSchemaVersion().orElse(signal.getLatestSchemaVersion())); + if (value.isPresent()) { + return ProtocolFactory.newHeadersWithJsonContentType(signal.getDittoHeaders()); + } else { + return signal.getDittoHeaders(); + } + } + + @Override + TopicPath getTopicPath(final PolicyEvent signal, final TopicPath.Channel channel) { + final EventsTopicPathBuilder topicPathBuilder = getEventsTopicPathBuilderOrThrow(signal, channel); + final String eventName = getLowerCaseEventName(signal); + if (isAction(eventName, TopicPath.Action.CREATED)) { + topicPathBuilder.created(); + } else if (isAction(eventName, TopicPath.Action.MODIFIED)) { + topicPathBuilder.modified(); + } else if (isAction(eventName, TopicPath.Action.DELETED)) { + topicPathBuilder.deleted(); + } else if (isAction(eventName, TopicPath.Action.MERGED)) { + topicPathBuilder.merged(); + } else { + throw UnknownEventException.newBuilder(eventName).build(); + } + return topicPathBuilder.build(); + } + + private static EventsTopicPathBuilder getEventsTopicPathBuilderOrThrow(final PolicyEvent event, + final TopicPath.Channel channel) { + + TopicPathBuilder topicPathBuilder = ProtocolFactory.newTopicPathBuilder(event.getEntityId()); + if (TopicPath.Channel.NONE == channel) { + topicPathBuilder = topicPathBuilder.none(); + } else { + throw UnknownChannelException.newBuilder(channel, event.getType()) + .dittoHeaders(event.getDittoHeaders()) + .build(); + } + return topicPathBuilder.events(); + } + + private static String getLowerCaseEventName(final PolicyEvent thingEvent) { + final Class thingEventClass = thingEvent.getClass(); + final String eventClassSimpleName = thingEventClass.getSimpleName(); + return eventClassSimpleName.toLowerCase(Locale.ENGLISH); + } + + private static boolean isAction(final String eventName, final TopicPath.Action expectedAction) { + return eventName.contains(expectedAction.getName()); + } +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/SignalMapperFactory.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/SignalMapperFactory.java index 81ad891cd9e..40f25b25f2c 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/SignalMapperFactory.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/SignalMapperFactory.java @@ -12,6 +12,8 @@ */ package org.eclipse.ditto.protocol.mapper; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.connectivity.model.signals.announcements.ConnectivityAnnouncement; import org.eclipse.ditto.messages.model.signals.commands.MessageCommand; import org.eclipse.ditto.messages.model.signals.commands.MessageCommandResponse; @@ -20,6 +22,7 @@ import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommand; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.things.model.signals.commands.modify.MergeThing; import org.eclipse.ditto.things.model.signals.commands.modify.MergeThingResponse; import org.eclipse.ditto.things.model.signals.commands.modify.ThingModifyCommand; @@ -80,6 +83,14 @@ public static SignalMapper> newSubscriptionEventSignalMappe return new SubscriptionEventSignalMapper(); } + public static SignalMapper> newStreamingSubscriptionCommandSignalMapper() { + return new StreamingSubscriptionCommandSignalMapper<>(); + } + + public static SignalMapper> newStreamingSubscriptionEventSignalMapper() { + return new StreamingSubscriptionEventSignalMapper(); + } + public static SignalMapper newRetrieveThingsSignalMapper() { return new RetrieveThingsSignalMapper(); } @@ -112,6 +123,10 @@ public static SignalMapper> newPolicyAnnouncementSignalMap return new PolicyAnnouncementSignalMapper(); } + public static SignalMapper> newPolicyEventSignalMapper() { + return new PolicyEventSignalMapper(); + } + public static SignalMapper> newMessageCommandSignalMapper() { return MessageSignalMapper.getInstance(); } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionCommandSignalMapper.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionCommandSignalMapper.java new file mode 100644 index 00000000000..49e1e1b6e57 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionCommandSignalMapper.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mapper; + +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.protocol.PayloadBuilder; +import org.eclipse.ditto.protocol.ProtocolFactory; +import org.eclipse.ditto.protocol.StreamingTopicPathBuilder; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.TopicPathBuilder; +import org.eclipse.ditto.protocol.UnknownCommandException; + +/** + * Signal mapper implementation for {@link StreamingSubscriptionCommand}s. + * + * @param the type of the command + */ +final class StreamingSubscriptionCommandSignalMapper> + extends AbstractSignalMapper { + + @Override + TopicPath getTopicPath(final T command, final TopicPath.Channel channel) { + final TopicPathBuilder topicPathBuilder = getTopicPathBuilder(command); + final StreamingTopicPathBuilder streamingTopicPathBuilder = + fromTopicPathBuilderWithChannel(topicPathBuilder, channel); + setTopicPathAction(streamingTopicPathBuilder, command, getSupportedActions()); + return streamingTopicPathBuilder.build(); + } + + /** + * @return array of {@link org.eclipse.ditto.protocol.TopicPath.Action}s the implementation supports. + */ + public TopicPath.StreamingAction[] getSupportedActions() { + return new TopicPath.StreamingAction[]{ + TopicPath.StreamingAction.REQUEST, + TopicPath.StreamingAction.CANCEL, + TopicPath.StreamingAction.SUBSCRIBE_FOR_PERSISTED_EVENTS + }; + } + + @Override + void enhancePayloadBuilder(final T command, final PayloadBuilder payloadBuilder) { + + final JsonObjectBuilder payloadContentBuilder = JsonFactory.newObjectBuilder(); + if (command instanceof SubscribeForPersistedEvents) { + final SubscribeForPersistedEvents subscribeCommand = (SubscribeForPersistedEvents) command; + payloadContentBuilder + .set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_REVISION, + subscribeCommand.getFromHistoricalRevision()) + .set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_REVISION, + subscribeCommand.getToHistoricalRevision()); + subscribeCommand.getFromHistoricalTimestamp().ifPresent(fromTs -> + payloadContentBuilder.set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP, + fromTs.toString())); + subscribeCommand.getToHistoricalTimestamp().ifPresent(toTs -> + payloadContentBuilder.set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_TIMESTAMP, + toTs.toString())); + } else if (command instanceof CancelStreamingSubscription) { + final CancelStreamingSubscription cancelCommand = (CancelStreamingSubscription) command; + payloadContentBuilder + .set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, cancelCommand.getSubscriptionId()); + } else if (command instanceof RequestFromStreamingSubscription) { + final RequestFromStreamingSubscription requestCommand = (RequestFromStreamingSubscription) command; + payloadContentBuilder + .set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, requestCommand.getSubscriptionId()) + .set(RequestFromStreamingSubscription.JsonFields.DEMAND, requestCommand.getDemand()); + } else { + throw UnknownCommandException.newBuilder(command.getClass().toString()).build(); + } + payloadBuilder.withValue(payloadContentBuilder.build()); + } + + private static StreamingTopicPathBuilder fromTopicPathBuilderWithChannel(final TopicPathBuilder topicPathBuilder, + final TopicPath.Channel channel) { + + if (channel == TopicPath.Channel.TWIN) { + return topicPathBuilder.twin().streaming(); + } else if (channel == TopicPath.Channel.NONE) { + return topicPathBuilder.none().streaming(); + } else { + throw new IllegalArgumentException("Unknown or unsupported Channel '" + channel + "'"); + } + } + + private TopicPathBuilder getTopicPathBuilder(final StreamingSubscriptionCommand command) { + return ProtocolFactory.newTopicPathBuilder(command.getEntityId()); + } + + private void setTopicPathAction(final StreamingTopicPathBuilder builder, final T command, + final TopicPath.StreamingAction... supportedActions) { + + if (supportedActions.length > 0) { + final String streamingCommandName = command.getName(); + final TopicPath.StreamingAction streamingAction = + TopicPath.StreamingAction.forName(streamingCommandName) + .orElseThrow(() -> unknownCommandException(streamingCommandName)); + setAction(builder, streamingAction); + } + } + + DittoRuntimeException unknownCommandException(final String commandName) { + return UnknownCommandException.newBuilder(commandName).build(); + } + + private void setAction(final StreamingTopicPathBuilder builder, final TopicPath.StreamingAction streamingAction) { + switch (streamingAction) { + case SUBSCRIBE_FOR_PERSISTED_EVENTS: + builder.subscribe(SubscribeForPersistedEvents.NAME); + break; + case REQUEST: + builder.request(); + break; + case CANCEL: + builder.cancel(); + break; + default: + throw unknownCommandException(streamingAction.getName()); + } + } +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionEventSignalMapper.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionEventSignalMapper.java new file mode 100644 index 00000000000..65755188cdf --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionEventSignalMapper.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mapper; + +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionCreated; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionHasNext; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.policies.model.PolicyConstants; +import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.protocol.PayloadBuilder; +import org.eclipse.ditto.protocol.ProtocolFactory; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.TopicPathBuilder; +import org.eclipse.ditto.protocol.UnknownEventException; +import org.eclipse.ditto.things.model.ThingConstants; +import org.eclipse.ditto.things.model.ThingId; + +/** + * Signal mapper implementation for {@link StreamingSubscriptionEvent}s. + */ +final class StreamingSubscriptionEventSignalMapper extends AbstractSignalMapper> { + + private static final JsonFieldDefinition SUBSCRIPTION_ID = + WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID; + + @Override + void enhancePayloadBuilder(final StreamingSubscriptionEvent signal, final PayloadBuilder payloadBuilder) { + final JsonObjectBuilder payloadContentBuilder = JsonFactory.newObjectBuilder(); + if (signal instanceof StreamingSubscriptionCreated) { + final StreamingSubscriptionCreated createdEvent = (StreamingSubscriptionCreated) signal; + payloadContentBuilder.set(SUBSCRIPTION_ID, createdEvent.getSubscriptionId()); + + } else if (signal instanceof StreamingSubscriptionComplete) { + final StreamingSubscriptionComplete completedEvent = (StreamingSubscriptionComplete) signal; + payloadContentBuilder.set(SUBSCRIPTION_ID, completedEvent.getSubscriptionId()); + + } else if (signal instanceof StreamingSubscriptionFailed) { + final StreamingSubscriptionFailed failedEvent = (StreamingSubscriptionFailed) signal; + payloadContentBuilder + .set(SUBSCRIPTION_ID, failedEvent.getSubscriptionId()) + .set(StreamingSubscriptionFailed.JsonFields.ERROR, failedEvent.getError().toJson()); + + } else if (signal instanceof StreamingSubscriptionHasNext) { + final StreamingSubscriptionHasNext hasNext = (StreamingSubscriptionHasNext) signal; + payloadContentBuilder + .set(SUBSCRIPTION_ID, hasNext.getSubscriptionId()) + .set(StreamingSubscriptionHasNext.JsonFields.ITEM, hasNext.getItem()); + + } else { + throw UnknownEventException.newBuilder(signal.getClass().getCanonicalName()).build(); + } + payloadBuilder.withValue(payloadContentBuilder.build()); + } + + @Override + DittoHeaders enhanceHeaders(final StreamingSubscriptionEvent signal) { + return ProtocolFactory.newHeadersWithJsonContentType(signal.getDittoHeaders()); + } + + @Override + TopicPath getTopicPath(final StreamingSubscriptionEvent signal, final TopicPath.Channel channel) { + + final TopicPathBuilder topicPathBuilder; + if (signal.getEntityType().equals(ThingConstants.ENTITY_TYPE)) { + topicPathBuilder = TopicPath.newBuilder(ThingId.of(signal.getEntityId())).things().twin(); + } else if (signal.getEntityType().equals(PolicyConstants.ENTITY_TYPE)) { + topicPathBuilder = TopicPath.newBuilder(PolicyId.of(signal.getEntityId())).policies().none(); + } else if (signal.getEntityType().equals(ConnectivityConstants.ENTITY_TYPE)) { + topicPathBuilder = TopicPath.newBuilder(ConnectionId.of(signal.getEntityId())).connections().none(); + } else { + throw UnknownEventException.newBuilder(signal.getClass().getCanonicalName()).build(); + } + + final TopicPath topicPath; + if (signal instanceof StreamingSubscriptionCreated) { + topicPath = topicPathBuilder.streaming().generated().build(); + } else if (signal instanceof StreamingSubscriptionComplete) { + topicPath = topicPathBuilder.streaming().complete().build(); + } else if (signal instanceof StreamingSubscriptionFailed) { + topicPath = topicPathBuilder.streaming().failed().build(); + } else if (signal instanceof StreamingSubscriptionHasNext) { + topicPath = topicPathBuilder.streaming().hasNext().build(); + } else { + throw UnknownEventException.newBuilder(signal.getClass().getCanonicalName()).build(); + } + return topicPath; + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/ThingMergeSignalMapper.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/ThingMergeSignalMapper.java index be1f1556d05..289dac4564a 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/ThingMergeSignalMapper.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/ThingMergeSignalMapper.java @@ -23,7 +23,7 @@ final class ThingMergeSignalMapper extends AbstractModifySignalMapper> activatedSubjectsFrom(final Adaptable adaptable) { + final JsonObject value = getValueFromPayload(adaptable); + return SubjectsModifiedPartially.modifiedSubjectsFromJson( + value.getValueOrThrow(SubjectsModifiedPartially.JSON_MODIFIED_SUBJECTS) + ); + } + + /** + * Subjects that are modified indexed by their policy entry labels. + * + * @param adaptable the adaptable + * @return the subjects + */ + protected static Map> deletedSubjectIdsFrom(final Adaptable adaptable) { + final JsonObject value = getValueFromPayload(adaptable); + return SubjectsDeletedPartially.deletedSubjectsFromJson( + value.getValueOrThrow(SubjectsDeletedPartially.JSON_DELETED_SUBJECT_IDS) + ); + } + /** * @throws NullPointerException if the value is null. */ diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractStreamingSubscriptionMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractStreamingSubscriptionMappingStrategies.java new file mode 100644 index 00000000000..554cbf946d2 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractStreamingSubscriptionMappingStrategies.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mappingstrategies; + +import java.util.Map; +import java.util.Optional; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.json.Jsonifiable; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonMissingFieldException; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.JsonifiableMapper; +import org.eclipse.ditto.protocol.Payload; +import org.eclipse.ditto.protocol.TopicPath; + +/** + * Provides helper methods to map from {@link Adaptable}s to streaming subscription commands and events. + * + * @param the type of the mapped signals + */ +abstract class AbstractStreamingSubscriptionMappingStrategies> + extends AbstractMappingStrategies { + + protected AbstractStreamingSubscriptionMappingStrategies(final Map> mappingStrategies) { + super(mappingStrategies); + } + + protected static NamespacedEntityId entityIdFrom(final Adaptable adaptable) { + final TopicPath topicPath = adaptable.getTopicPath(); + return NamespacedEntityId.of(topicPath.getGroup().getEntityType(), + topicPath.getNamespace() + ":" + topicPath.getEntityName()); + } + + protected static EntityType entityTypeFrom(final Adaptable adaptable) { + final TopicPath topicPath = adaptable.getTopicPath(); + return topicPath.getGroup().getEntityType(); + } + + protected static JsonPointer resourcePathFrom(final Adaptable adaptable) { + return adaptable.getPayload().getPath(); + } + + protected static String subscriptionIdFrom(final Adaptable adaptable) { + return adaptable.getPayload().getValue() + .map(value -> value.asObject().getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID)) + .orElseThrow(() -> JsonMissingFieldException.newBuilder() + .fieldName(Payload.JsonFields.VALUE.getPointer()) + .build() + ); + } + + static Optional getFromValue(final Adaptable adaptable, final JsonFieldDefinition jsonFieldDefinition) { + return adaptable.getPayload().getValue().flatMap(value -> value.asObject().getValue(jsonFieldDefinition)); + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/MappingStrategiesFactory.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/MappingStrategiesFactory.java index 90da18949cf..22b7abf4fda 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/MappingStrategiesFactory.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/MappingStrategiesFactory.java @@ -43,6 +43,10 @@ public static PolicyAnnouncementMappingStrategies getPolicyAnnouncementMappingSt return PolicyAnnouncementMappingStrategies.getInstance(); } + public static PolicyEventMappingStrategies getPolicyEventMappingStrategies() { + return PolicyEventMappingStrategies.getInstance(); + } + public static ThingMergeCommandMappingStrategies getThingMergeCommandMappingStrategies() { return ThingMergeCommandMappingStrategies.getInstance(); } @@ -110,4 +114,13 @@ public static ConnectivityAnnouncementMappingStrategies getConnectivityAnnouncem return ConnectivityAnnouncementMappingStrategies.getInstance(); } + public static StreamingSubscriptionCommandMappingStrategies getStreamingSubscriptionCommandMappingStrategies() { + return StreamingSubscriptionCommandMappingStrategies.getInstance(); + } + + public static StreamingSubscriptionEventMappingStrategies getStreamingSubscriptionEventMappingStrategies( + final ErrorRegistry errorRegistry) { + return StreamingSubscriptionEventMappingStrategies.getInstance(errorRegistry); + } + } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/PolicyEventMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/PolicyEventMappingStrategies.java new file mode 100644 index 00000000000..0dee6ad0aab --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/PolicyEventMappingStrategies.java @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mappingstrategies; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.json.JsonMissingFieldException; +import org.eclipse.ditto.policies.model.signals.events.PolicyCreated; +import org.eclipse.ditto.policies.model.signals.events.PolicyDeleted; +import org.eclipse.ditto.policies.model.signals.events.PolicyEntriesModified; +import org.eclipse.ditto.policies.model.signals.events.PolicyEntryCreated; +import org.eclipse.ditto.policies.model.signals.events.PolicyEntryDeleted; +import org.eclipse.ditto.policies.model.signals.events.PolicyEntryModified; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.policies.model.signals.events.PolicyModified; +import org.eclipse.ditto.policies.model.signals.events.ResourceCreated; +import org.eclipse.ditto.policies.model.signals.events.ResourceDeleted; +import org.eclipse.ditto.policies.model.signals.events.ResourceModified; +import org.eclipse.ditto.policies.model.signals.events.ResourcesModified; +import org.eclipse.ditto.policies.model.signals.events.SubjectCreated; +import org.eclipse.ditto.policies.model.signals.events.SubjectDeleted; +import org.eclipse.ditto.policies.model.signals.events.SubjectModified; +import org.eclipse.ditto.policies.model.signals.events.SubjectsDeletedPartially; +import org.eclipse.ditto.policies.model.signals.events.SubjectsModified; +import org.eclipse.ditto.policies.model.signals.events.SubjectsModifiedPartially; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.JsonifiableMapper; +import org.eclipse.ditto.protocol.Payload; + +/** + * Defines mapping strategies (map from signal type to JsonifiableMapper) for policy events. + */ +final class PolicyEventMappingStrategies extends AbstractPolicyMappingStrategies> { + + private static final PolicyEventMappingStrategies INSTANCE = new PolicyEventMappingStrategies(); + + private PolicyEventMappingStrategies() { + super(initMappingStrategies()); + } + + static PolicyEventMappingStrategies getInstance() { + return INSTANCE; + } + + private static Map>> initMappingStrategies() { + final Map>> mappingStrategies = new HashMap<>(); + addTopLevelEvents(mappingStrategies); + addPolicyEntriesEvents(mappingStrategies); + addPolicyEntryEvents(mappingStrategies); + addResourcesEvents(mappingStrategies); + addResourceEvents(mappingStrategies); + addSubjectsEvents(mappingStrategies); + addSubjectEvents(mappingStrategies); + return mappingStrategies; + } + + private static void addTopLevelEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(PolicyCreated.TYPE, + adaptable -> PolicyCreated.of(policyFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(PolicyModified.TYPE, + adaptable -> PolicyModified.of(policyFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(PolicyDeleted.TYPE, + adaptable -> PolicyDeleted.of(policyIdFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addPolicyEntriesEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(PolicyEntriesModified.TYPE, + adaptable -> PolicyEntriesModified.of(policyIdFrom(adaptable), + policyEntriesFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addPolicyEntryEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(PolicyEntryCreated.TYPE, + adaptable -> PolicyEntryCreated.of(policyIdFrom(adaptable), + policyEntryFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(PolicyEntryModified.TYPE, + adaptable -> PolicyEntryModified.of(policyIdFrom(adaptable), + policyEntryFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(PolicyEntryDeleted.TYPE, + adaptable -> PolicyEntryDeleted.of(policyIdFrom(adaptable), + labelFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addResourcesEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(ResourcesModified.TYPE, + adaptable -> ResourcesModified.of(policyIdFrom(adaptable), + labelFrom(adaptable), + resourcesFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addResourceEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(ResourceCreated.TYPE, + adaptable -> ResourceCreated.of(policyIdFrom(adaptable), + labelFrom(adaptable), + resourceFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(ResourceModified.TYPE, + adaptable -> ResourceModified.of(policyIdFrom(adaptable), + labelFrom(adaptable), + resourceFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(ResourceDeleted.TYPE, + adaptable -> ResourceDeleted.of(policyIdFrom(adaptable), + labelFrom(adaptable), + entryResourceKeyFromPath(adaptable.getPayload().getPath()), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addSubjectsEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(SubjectsModified.TYPE, + adaptable -> SubjectsModified.of(policyIdFrom(adaptable), + labelFrom(adaptable), + subjectsFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(SubjectsModifiedPartially.TYPE, + adaptable -> SubjectsModifiedPartially.of(policyIdFrom(adaptable), + activatedSubjectsFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(SubjectsDeletedPartially.TYPE, + adaptable -> SubjectsDeletedPartially.of(policyIdFrom(adaptable), + deletedSubjectIdsFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addSubjectEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(SubjectCreated.TYPE, + adaptable -> SubjectCreated.of(policyIdFrom(adaptable), + labelFrom(adaptable), + subjectFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(SubjectModified.TYPE, + adaptable -> SubjectModified.of(policyIdFrom(adaptable), + labelFrom(adaptable), + subjectFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(SubjectDeleted.TYPE, + adaptable -> SubjectDeleted.of(policyIdFrom(adaptable), + labelFrom(adaptable), + entrySubjectIdFromPath(adaptable.getPayload().getPath()), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static long revisionFrom(final Adaptable adaptable) { + return adaptable.getPayload().getRevision().orElseThrow(() -> JsonMissingFieldException.newBuilder() + .fieldName(Payload.JsonFields.REVISION.getPointer().toString()).build()); + } + + @Nullable + private static Instant timestampFrom(final Adaptable adaptable) { + return adaptable.getPayload().getTimestamp().orElse(null); + } + + @Nullable + private static Metadata metadataFrom(final Adaptable adaptable) { + return adaptable.getPayload().getMetadata().orElse(null); + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionCommandMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionCommandMappingStrategies.java new file mode 100644 index 00000000000..fb8881e7eb3 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionCommandMappingStrategies.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mappingstrategies; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.JsonifiableMapper; + +/** + * Defines mapping strategies (map from signal type to JsonifiableMapper) for streaming subscription commands. + */ +final class StreamingSubscriptionCommandMappingStrategies + extends AbstractStreamingSubscriptionMappingStrategies> { + + private static final StreamingSubscriptionCommandMappingStrategies INSTANCE = + new StreamingSubscriptionCommandMappingStrategies(); + + private StreamingSubscriptionCommandMappingStrategies() { + super(initMappingStrategies()); + } + + static StreamingSubscriptionCommandMappingStrategies getInstance() { + return INSTANCE; + } + + private static Map>> initMappingStrategies() { + + final Map>> mappingStrategies = new HashMap<>(); + + mappingStrategies.put(SubscribeForPersistedEvents.TYPE, + adaptable -> SubscribeForPersistedEvents.of(entityIdFrom(adaptable), + resourcePathFrom(adaptable), + fromHistoricalRevision(adaptable), + toHistoricalRevision(adaptable), + fromHistoricalTimestamp(adaptable), + toHistoricalTimestamp(adaptable), + dittoHeadersFrom(adaptable))); + mappingStrategies.put(CancelStreamingSubscription.TYPE, + adaptable -> CancelStreamingSubscription.of(entityIdFrom(adaptable), + resourcePathFrom(adaptable), + Objects.requireNonNull(subscriptionIdFrom(adaptable)), + dittoHeadersFrom(adaptable))); + mappingStrategies.put(RequestFromStreamingSubscription.TYPE, + adaptable -> RequestFromStreamingSubscription.of(entityIdFrom(adaptable), + resourcePathFrom(adaptable), + Objects.requireNonNull(subscriptionIdFrom(adaptable)), + demandFrom(adaptable), + dittoHeadersFrom(adaptable))); + + return mappingStrategies; + } + + @Nullable + private static Long fromHistoricalRevision(final Adaptable adaptable) { + return getFromValue(adaptable, SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_REVISION) + .orElse(null); + } + + @Nullable + private static Long toHistoricalRevision(final Adaptable adaptable) { + return getFromValue(adaptable, SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_REVISION) + .orElse(null); + } + + @Nullable + private static Instant fromHistoricalTimestamp(final Adaptable adaptable) { + return getFromValue(adaptable, SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP) + .map(Instant::parse) + .orElse(null); + } + + @Nullable + private static Instant toHistoricalTimestamp(final Adaptable adaptable) { + return getFromValue(adaptable, SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_TIMESTAMP) + .map(Instant::parse) + .orElse(null); + } + + private static long demandFrom(final Adaptable adaptable) { + return getFromValue(adaptable, RequestFromStreamingSubscription.JsonFields.DEMAND).orElse(0L); + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionEventMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionEventMappingStrategies.java new file mode 100644 index 00000000000..07bfa9abe56 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionEventMappingStrategies.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mappingstrategies; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.ErrorRegistry; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionCreated; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionHasNext; +import org.eclipse.ditto.json.JsonParseException; +import org.eclipse.ditto.json.JsonValue; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.JsonifiableMapper; +import org.eclipse.ditto.protocol.adapter.AbstractErrorResponseAdapter; + +/** + * Defines mapping strategies (map from signal type to JsonifiableMapper) for streaming subscription events. + */ +final class StreamingSubscriptionEventMappingStrategies + extends AbstractStreamingSubscriptionMappingStrategies> { + + private StreamingSubscriptionEventMappingStrategies(final ErrorRegistry errorRegistry) { + super(initMappingStrategies(errorRegistry)); + } + + static StreamingSubscriptionEventMappingStrategies getInstance(final ErrorRegistry errorRegistry) { + return new StreamingSubscriptionEventMappingStrategies(errorRegistry); + } + + private static Map>> initMappingStrategies( + final ErrorRegistry errorRegistry) { + + final Map>> mappingStrategies = new HashMap<>(); + + mappingStrategies.put(StreamingSubscriptionCreated.TYPE, + adaptable -> StreamingSubscriptionCreated.of(Objects.requireNonNull(subscriptionIdFrom(adaptable)), + entityIdFrom(adaptable), dittoHeadersFrom(adaptable))); + mappingStrategies.put(StreamingSubscriptionComplete.TYPE, + adaptable -> StreamingSubscriptionComplete.of(Objects.requireNonNull(subscriptionIdFrom(adaptable)), + entityIdFrom(adaptable), dittoHeadersFrom(adaptable))); + mappingStrategies.put(StreamingSubscriptionFailed.TYPE, + adaptable -> StreamingSubscriptionFailed.of(Objects.requireNonNull(subscriptionIdFrom(adaptable)), + entityIdFrom(adaptable), errorFrom(adaptable, errorRegistry), dittoHeadersFrom(adaptable))); + mappingStrategies.put(StreamingSubscriptionHasNext.TYPE, + adaptable -> StreamingSubscriptionHasNext.of(Objects.requireNonNull(subscriptionIdFrom(adaptable)), + entityIdFrom(adaptable),itemFrom(adaptable), dittoHeadersFrom(adaptable))); + + return mappingStrategies; + } + + private static JsonValue itemFrom(final Adaptable adaptable) { + return getFromValue(adaptable, StreamingSubscriptionHasNext.JsonFields.ITEM).orElseGet(JsonValue::nullLiteral); + } + + private static DittoRuntimeException errorFrom(final Adaptable adaptable, final ErrorRegistry errorRegistry) { + return getFromValue(adaptable, StreamingSubscriptionFailed.JsonFields.ERROR) + .map(error -> + AbstractErrorResponseAdapter.parseWithErrorRegistry(error, DittoHeaders.empty(), errorRegistry)) + .orElseThrow(() -> JsonParseException.newBuilder().build()); + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/ThingMergeCommandMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/ThingMergeCommandMappingStrategies.java index 82c755572aa..54826810938 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/ThingMergeCommandMappingStrategies.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/ThingMergeCommandMappingStrategies.java @@ -16,19 +16,22 @@ import java.util.Map; import java.util.Optional; -import org.eclipse.ditto.json.JsonPointer; -import org.eclipse.ditto.json.JsonValue; +import javax.annotation.Nullable; + import org.eclipse.ditto.base.model.exceptions.UnsupportedMediaTypeException; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.contenttype.ContentType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.json.JsonValue; import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.JsonifiableMapper; import org.eclipse.ditto.things.model.Attributes; import org.eclipse.ditto.things.model.Thing; import org.eclipse.ditto.things.model.ThingDefinition; import org.eclipse.ditto.things.model.ThingsModelFactory; -import org.eclipse.ditto.protocol.Adaptable; -import org.eclipse.ditto.protocol.JsonifiableMapper; import org.eclipse.ditto.things.model.signals.commands.exceptions.PolicyIdNotDeletableException; import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingIdNotDeletableException; import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingMergeInvalidException; @@ -70,7 +73,9 @@ private static Map> initMappingStrategies( private static MergeThing mergeThing(final Adaptable adaptable) { checkContentTypeHeader(adaptable.getDittoHeaders()); - return MergeThing.withThing(thingIdFrom(adaptable), thingForMergeFrom(adaptable), dittoHeadersFrom(adaptable)); + return MergeThing.withThing(thingIdFrom(adaptable), thingForMergeFrom(adaptable), + initialPolicyForMergeThingFrom(adaptable), policyIdOrPlaceholderForMergeThingFrom(adaptable), + dittoHeadersFrom(adaptable)); } private static MergeThing mergeThingWithPolicyId(final Adaptable adaptable) { @@ -226,4 +231,23 @@ private static void checkContentTypeHeader(final DittoHeaders dittoHeaders) { } } + @Nullable + private static JsonObject initialPolicyForMergeThingFrom(final Adaptable adaptable) { + return adaptable.getPayload() + .getValue() + .filter(JsonValue::isObject) + .map(JsonValue::asObject) + .flatMap(MergeThing::initialPolicyForMergeThingFrom) + .orElse(null); + } + + @Nullable + private static String policyIdOrPlaceholderForMergeThingFrom(final Adaptable adaptable) { + return adaptable.getPayload() + .getValue() + .filter(JsonValue::isObject) + .map(JsonValue::asObject) + .flatMap(MergeThing::policyIdOrPlaceholderForMergeThingFrom) + .orElse(null); + } } diff --git a/protocol/src/test/java/org/eclipse/ditto/protocol/ImmutableMessagePathTest.java b/protocol/src/test/java/org/eclipse/ditto/protocol/ImmutableMessagePathTest.java index 10071f659b2..9aa90053850 100644 --- a/protocol/src/test/java/org/eclipse/ditto/protocol/ImmutableMessagePathTest.java +++ b/protocol/src/test/java/org/eclipse/ditto/protocol/ImmutableMessagePathTest.java @@ -20,6 +20,7 @@ import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; +import org.assertj.core.api.AbstractBooleanAssert; import org.assertj.core.api.OptionalAssert; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.messages.model.MessageDirection; @@ -68,6 +69,23 @@ public void parseFeatureId() { assertFeatureId("features/water-tank/inbox/messages/heatUp").contains("water-tank"); } + @Test + public void parseMessageSubject() { + assertMessageSubject("/outbox/message/ask").contains("ask"); + assertMessageSubject("/attributes/hello").isEmpty(); + assertMessageSubject("/features/water-tank/properties/temperature").isEmpty(); + assertMessageSubject("features/water-tank/inbox/messages/heatUp").contains("heatUp"); + assertMessageSubject("/features/water-tank/inbox/messages/heatUp/subMsg").contains("heatUp/subMsg"); + } + + @Test + public void parseIsInboxOutboxMessage() { + assertIsInboxOutboxMessage("/outbox/message/ask").isTrue(); + assertIsInboxOutboxMessage("/attributes/hello").isFalse(); + assertIsInboxOutboxMessage("/features/water-tank/properties/temperature").isFalse(); + assertIsInboxOutboxMessage("features/water-tank/inbox/messages/heatUp").isTrue(); + } + private static OptionalAssert assertDirection(final String jsonPointer) { return assertThat(ImmutableMessagePath.of(JsonPointer.of(jsonPointer)).getDirection()); } @@ -75,4 +93,12 @@ private static OptionalAssert assertDirection(final String jso private static OptionalAssert assertFeatureId(final String jsonPointer) { return assertThat(ImmutableMessagePath.of(JsonPointer.of(jsonPointer)).getFeatureId()); } + + private static OptionalAssert assertMessageSubject(final String jsonPointer) { + return assertThat(ImmutableMessagePath.of(JsonPointer.of(jsonPointer)).getMessageSubject()); + } + + private static AbstractBooleanAssert assertIsInboxOutboxMessage(final String jsonPointer) { + return assertThat(ImmutableMessagePath.of(JsonPointer.of(jsonPointer)).isInboxOutboxMessage()); + } } diff --git a/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapterParameterizedTest.java b/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapterParameterizedTest.java index cae6597b5fa..c99d8507e33 100644 --- a/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapterParameterizedTest.java +++ b/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapterParameterizedTest.java @@ -212,6 +212,11 @@ public void setUp() { final ConnectivityCommandAdapterProvider connectivityCommandAdapterProvider = mock(ConnectivityCommandAdapterProvider.class); + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter = + mock(StreamingSubscriptionCommandAdapter.class); + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter = + mock(StreamingSubscriptionEventAdapter.class); + when(thingCommandAdapterProvider.getQueryCommandAdapter()) .thenReturn(thingQueryCommandAdapter); when(thingCommandAdapterProvider.getQueryCommandResponseAdapter()) @@ -247,9 +252,11 @@ public void setUp() { .thenReturn(policyErrorResponseAdapter); final AdapterResolver adapterResolver = new DefaultAdapterResolver(thingCommandAdapterProvider, - policyCommandAdapterProvider, connectivityCommandAdapterProvider, acknowledgementAdapterProvider); + policyCommandAdapterProvider, connectivityCommandAdapterProvider, acknowledgementAdapterProvider, + streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter); underTest = DittoProtocolAdapter.newInstance(HeaderTranslator.empty(), thingCommandAdapterProvider, policyCommandAdapterProvider, connectivityCommandAdapterProvider, acknowledgementAdapterProvider, + streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter, adapterResolver); reset(thingQueryCommandAdapter); diff --git a/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/things/ThingMergeCommandAdapterTest.java b/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/things/ThingMergeCommandAdapterTest.java index 8714dcd69ae..24c6d339d6f 100644 --- a/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/things/ThingMergeCommandAdapterTest.java +++ b/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/things/ThingMergeCommandAdapterTest.java @@ -15,21 +15,22 @@ import static org.assertj.core.api.Assertions.assertThat; import org.assertj.core.api.Assertions; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonPointer; -import org.eclipse.ditto.json.JsonValue; import org.eclipse.ditto.base.model.exceptions.DittoJsonException; import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; import org.eclipse.ditto.base.model.headers.contenttype.ContentType; -import org.eclipse.ditto.things.model.Thing; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.json.JsonValue; import org.eclipse.ditto.protocol.Adaptable; -import org.eclipse.ditto.protocol.adapter.DittoProtocolAdapter; import org.eclipse.ditto.protocol.LiveTwinTest; import org.eclipse.ditto.protocol.Payload; -import org.eclipse.ditto.protocol.adapter.ProtocolAdapterTest; import org.eclipse.ditto.protocol.TestConstants; import org.eclipse.ditto.protocol.TopicPath; import org.eclipse.ditto.protocol.UnknownPathException; +import org.eclipse.ditto.protocol.adapter.DittoProtocolAdapter; +import org.eclipse.ditto.protocol.adapter.ProtocolAdapterTest; +import org.eclipse.ditto.things.model.Thing; import org.eclipse.ditto.things.model.signals.commands.exceptions.PolicyIdNotDeletableException; import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingIdNotDeletableException; import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingIdNotExplicitlySettableException; @@ -133,6 +134,28 @@ public void mergeThingWithPolicyIdFromAdaptable() { assertWithExternalHeadersThat(actual).isEqualTo(expected); } + @Test + public void mergeThingWithInlinePolicyFromAdaptable() { + final JsonObject initialPolicy = JsonObject.newBuilder().set("foo", "bar").build(); + final MergeThing expected = + MergeThing.withThing(TestConstants.THING_ID, TestConstants.THING, initialPolicy, null, + TestConstants.DITTO_HEADERS_V_2); + final Adaptable adaptable = Adaptable.newBuilder(topicPath) + .withPayload(Payload.newBuilder(JsonPointer.empty()) + .withValue(TestConstants.THING.toJson().toBuilder() + .set(MergeThing.JSON_INLINE_POLICY.getPointer(), initialPolicy) + .build() + ) + .build() + ) + .withHeaders(TestConstants.HEADERS_V_2_FOR_MERGE_COMMANDS) + .build(); + + final MergeThing actual = underTest.fromAdaptable(adaptable); + + assertWithExternalHeadersThat(actual).isEqualTo(expected); + } + @Test(expected = PolicyIdNotDeletableException.class) public void mergeThingWithNullPolicyIdFromAdaptable() { final Adaptable adaptable = Adaptable.newBuilder(topicPath) diff --git a/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java b/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java index b8359d0cc90..421947b8536 100755 --- a/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java +++ b/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java @@ -76,9 +76,14 @@ public enum Type { LE("le"), /** - * Represents a lower than or equals comparison. + * Represents a string 'like' comparison, supporting wildcards '*' for multiple and '?' for a single character. + */ + LIKE("like"), + + /** + * Represents a string 'like' comparison, supporting wildcards '*' for multiple and '?' for a single character with case sensitivity. */ - LIKE("like"); + ILIKE("ilike"); private final String name; diff --git a/rql/parser/src/main/scala/org/eclipse/ditto/rql/parser/internal/RqlPredicateParser.scala b/rql/parser/src/main/scala/org/eclipse/ditto/rql/parser/internal/RqlPredicateParser.scala index b0312f2a718..4da7f37112e 100644 --- a/rql/parser/src/main/scala/org/eclipse/ditto/rql/parser/internal/RqlPredicateParser.scala +++ b/rql/parser/src/main/scala/org/eclipse/ditto/rql/parser/internal/RqlPredicateParser.scala @@ -28,7 +28,7 @@ import scala.util.{Failure, Success} * 

       * Query                      = SingleComparisonOp | MultiComparisonOp | MultiLogicalOp | SingleLogicalOp | ExistsOp
   * SingleComparisonOp         = SingleComparisonName, '(', ComparisonProperty, ',', ComparisonValue, ')'
-  * SingleComparisonName       = "eq" | "ne" | "gt" | "ge" | "lt" | "le" | "like"
+  * SingleComparisonName       = "eq" | "ne" | "gt" | "ge" | "lt" | "le" | "like" | "ilike"
   * MultiComparisonOp          = MultiComparisonName, '(', ComparisonProperty, ',', ComparisonValue, { ',', ComparisonValue }, ')'
   * MultiComparisonName        = "in"
   * MultiLogicalOp             = MultiLogicalName, '(', Query, { ',', Query }, ')'
@@ -71,10 +71,10 @@ private class RqlPredicateParser(override val input: ParserInput) extends RqlPar
   }
 
   /**
-    * SingleComparisonName       = "eq" | "ne" | "gt" | "ge" | "lt" | "le" | "like"
+    * SingleComparisonName       = "eq" | "ne" | "gt" | "ge" | "lt" | "le" | "like" | "ilike"
     */
   private def SingleComparisonName: Rule1[SingleComparisonNode.Type] = rule {
-    eq | ne | gt | ge | lt | le | like
+    eq | ne | gt | ge | lt | le | like | ilike
   }
 
   private def eq: Rule1[SingleComparisonNode.Type] = rule {
@@ -105,6 +105,10 @@ private class RqlPredicateParser(override val input: ParserInput) extends RqlPar
     "like" ~ push(SingleComparisonNode.Type.LIKE)
   }
 
+  private def ilike: Rule1[SingleComparisonNode.Type] = rule {
+    "ilike" ~ push(SingleComparisonNode.Type.ILIKE)
+  }
+
   /**
     * MultiComparisonOp          = MultiComparisonName, '(', ComparisonProperty, ',', ComparisonValue, { ',', ComparisonValue }, ')'
     */
diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java
index cc78834d5ae..f27a67e2a6e 100644
--- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java
+++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java
@@ -145,13 +145,21 @@ default Criteria nor(final Criteria criteria) {
     Predicate le(@Nullable Object value);
 
     /**
-     * Creates a predicate which checks lower than or equals.
+     * Represents a string 'like' comparison, supporting wildcards '*' for multiple and '?' for a single character.
      *
      * @param value the value, may be {@code null}.
      * @return the predicate.
      */
     Predicate like(@Nullable Object value);
 
+    /**
+     * Represents a string 'like' comparison, supporting wildcards '*' for multiple and '?' for a single character with case insensitivity.
+     * @since 3.2.0
+     * @param value the value, may be {@code null}.
+     * @return the predicate. 
+     */
+    Predicate ilike(@Nullable Object value);
+    
     /**
      * The $in predicate selects the documents where the value of a field equals any value in the specified array.
      *
diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java
index b1a195c62a0..02de81067c2 100644
--- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java
+++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java
@@ -110,6 +110,15 @@ public Predicate like(@Nullable final Object value) {
         }
     }
 
+    @Override
+    public Predicate ilike(@Nullable final Object value) {
+        if (value instanceof String) {
+            return new ILikePredicateImpl(value);
+        } else {
+            throw new IllegalArgumentException("In the ilike predicate only string values are allowed.");
+        }
+    }
+
     @Override
     public Predicate in(final List values) {
         return new InPredicateImpl(requireNonNull(values));
diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java
new file mode 100644
index 00000000000..19cd596252c
--- /dev/null
+++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.ditto.rql.query.criteria;
+
+import javax.annotation.Nullable;
+
+import org.eclipse.ditto.base.model.common.LikeHelper;
+import org.eclipse.ditto.rql.query.criteria.visitors.PredicateVisitor;
+
+/**
+ * ILike predicate.
+ */
+final class ILikePredicateImpl extends AbstractSinglePredicate {
+
+    public ILikePredicateImpl(@Nullable final Object value) {
+        super(value);
+    }
+
+    @Nullable
+    private String convertToRegexSyntaxAndGetOption() {
+        final Object value = getValue();
+        if (value != null) {
+            return LikeHelper.convertToRegexSyntax(value.toString());
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public  T accept(final PredicateVisitor visitor) {
+        return visitor.visitILike(convertToRegexSyntaxAndGetOption());
+    }
+}
diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java
index 868d2245ed4..cc016f33594 100644
--- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java
+++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java
@@ -37,6 +37,8 @@ public interface PredicateVisitor {
 
     T visitLike(@Nullable String value);
 
+    T visitILike(@Nullable String value);
+
     T visitIn(List values);
 
 }
diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/filter/ParameterPredicateVisitor.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/filter/ParameterPredicateVisitor.java
index a73b7ec2ad9..90acb7e0b9b 100644
--- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/filter/ParameterPredicateVisitor.java
+++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/filter/ParameterPredicateVisitor.java
@@ -54,6 +54,7 @@ final class ParameterPredicateVisitor implements PredicateVisitor {
         SINGLE_COMPARISON_NODE_MAPPING.put(SingleComparisonNode.Type.LT, CriteriaFactory::lt);
         SINGLE_COMPARISON_NODE_MAPPING.put(SingleComparisonNode.Type.LE, CriteriaFactory::le);
         SINGLE_COMPARISON_NODE_MAPPING.put(SingleComparisonNode.Type.LIKE, CriteriaFactory::like);
+        SINGLE_COMPARISON_NODE_MAPPING.put(SingleComparisonNode.Type.ILIKE, CriteriaFactory::ilike);
     }
 
     private final List criteria = new ArrayList<>();
diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java
index fdcda2c60a6..3fef861ef91 100644
--- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java
+++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java
@@ -241,6 +241,16 @@ public Function> visitLike(@Nullable final String value
                         .isPresent();
     }
 
+    @Override
+    public Function> visitILike(@Nullable final String value) {
+        return fieldName ->
+                thing -> getThingField(fieldName, thing)
+                        .filter(JsonValue::isString)
+                        .map(JsonValue::asString)
+                        .filter(str -> null != value && Pattern.compile(value, Pattern.CASE_INSENSITIVE).matcher(str).matches())
+                        .isPresent();
+    }
+    
     @Nullable
     private Object resolveValue(@Nullable final Object value) {
         if (value instanceof ParsedPlaceholder) {
diff --git a/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java b/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java
index 691f740d3c9..c27423ae570 100644
--- a/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java
+++ b/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java
@@ -137,6 +137,13 @@ public void matchingStringLike() {
                 .isTrue();
     }
 
+    @Test
+    public void matchingStringILike() {
+        // the sut already works on regex Pattern - the translation from "*" to ".*" followed by case insensitivity is done in LikePredicateImpl
+        doTest(sut.visitILike("this-is.*"), JsonValue.of("THIS-IS-THE-CONTENT"))
+                .isTrue();
+    }
+
     @Test
     public void matchingViaPlaceholderStringLike() {
         // the sut already works on regex Pattern - the translation from "*" to ".*" is done in LikePredicateImpl
diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/InvalidThingFieldSelectionException.java b/things/model/src/main/java/org/eclipse/ditto/things/model/InvalidThingFieldSelectionException.java
index 6b32b00180d..efcbd64f6c4 100644
--- a/things/model/src/main/java/org/eclipse/ditto/things/model/InvalidThingFieldSelectionException.java
+++ b/things/model/src/main/java/org/eclipse/ditto/things/model/InvalidThingFieldSelectionException.java
@@ -18,12 +18,12 @@
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.NotThreadSafe;
 
-import org.eclipse.ditto.json.JsonObject;
 import org.eclipse.ditto.base.model.common.HttpStatus;
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException;
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.base.model.json.JsonParsableException;
+import org.eclipse.ditto.json.JsonObject;
 
 /**
  * Thrown when a {@link ThingFieldSelector} was not valid.
@@ -34,9 +34,9 @@
 public final class InvalidThingFieldSelectionException extends DittoRuntimeException implements ThingException {
 
     private static final String DEFAULT_MESSAGE_TEMPLATE = "Thing field selection <{0}> was not valid.";
-    private static final String DEFAULT_DESCRIPTION = "Please provide a comma separated List of valid Thing fields." +
+    private static final String DEFAULT_DESCRIPTION = "Please provide a comma separated List of valid Thing fields. " +
             "Make sure that you did not use a space after a comma. Valid Fields are: " +
-            ThingFieldSelector.SELECTABLE_FIELDS.toString();
+            ThingFieldSelector.SELECTABLE_FIELDS;
 
     static final String ERROR_CODE = ERROR_CODE_PREFIX + "field.selection.invalid";
 
diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/ThingFieldSelector.java b/things/model/src/main/java/org/eclipse/ditto/things/model/ThingFieldSelector.java
index 835d8efd904..81c3e55e8a5 100644
--- a/things/model/src/main/java/org/eclipse/ditto/things/model/ThingFieldSelector.java
+++ b/things/model/src/main/java/org/eclipse/ditto/things/model/ThingFieldSelector.java
@@ -38,7 +38,7 @@ public final class ThingFieldSelector implements JsonFieldSelector {
             .withoutUrlDecoding()
             .build();
     static final List SELECTABLE_FIELDS = Arrays.asList("thingId", "policyId", "definition",
-            "_namespace", "_revision", "_created", "_modified", "_metadata", "_policy",
+            "_namespace", "_revision", "_created", "_modified", "_metadata", "_policy", "_context", "_context(/[^,]+)?",
             "features(/[^,]+)?", "attributes(/[^,]+)?");
     private static final String KNOWN_FIELDS_REGEX = "/?(" + String.join("|", SELECTABLE_FIELDS) + ")";
     private static final String FIELD_SELECTION_REGEX = "^" + KNOWN_FIELDS_REGEX + "(," + KNOWN_FIELDS_REGEX + ")*$";
diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/ThingIdInvalidException.java b/things/model/src/main/java/org/eclipse/ditto/things/model/ThingIdInvalidException.java
index 000980d1947..1c7799644b8 100755
--- a/things/model/src/main/java/org/eclipse/ditto/things/model/ThingIdInvalidException.java
+++ b/things/model/src/main/java/org/eclipse/ditto/things/model/ThingIdInvalidException.java
@@ -44,7 +44,7 @@ public final class ThingIdInvalidException extends EntityIdInvalidException impl
             "It must conform to the namespaced entity ID notation (see Ditto documentation)";
 
     private static final URI DEFAULT_HREF =
-            URI.create("https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id");
+            URI.create("https://www.eclipse.dev/ditto/basic-namespaces-and-names.html#namespaced-id");
 
     private static final long serialVersionUID = -2026814719409279158L;
 
diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/WithThingId.java b/things/model/src/main/java/org/eclipse/ditto/things/model/WithThingId.java
index 67c1ce14736..3b156c9d299 100644
--- a/things/model/src/main/java/org/eclipse/ditto/things/model/WithThingId.java
+++ b/things/model/src/main/java/org/eclipse/ditto/things/model/WithThingId.java
@@ -16,7 +16,7 @@
 
 /**
  * Implementations of this interface are associated to a {@code Thing} identified by the value
- * returned from {@link #getEntityId()} ()}.
+ * returned from {@link #getEntityId()}.
  */
 public interface WithThingId extends WithEntityId {
 
diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingHistoryNotAccessibleException.java b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingHistoryNotAccessibleException.java
new file mode 100755
index 00000000000..e06eb9e1a1e
--- /dev/null
+++ b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingHistoryNotAccessibleException.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.ditto.things.model.signals.commands.exceptions;
+
+import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull;
+
+import java.net.URI;
+import java.text.MessageFormat;
+import java.time.Instant;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.Immutable;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.eclipse.ditto.base.model.common.HttpStatus;
+import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException;
+import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder;
+import org.eclipse.ditto.base.model.headers.DittoHeaders;
+import org.eclipse.ditto.base.model.json.JsonParsableException;
+import org.eclipse.ditto.json.JsonObject;
+import org.eclipse.ditto.things.model.ThingException;
+import org.eclipse.ditto.things.model.ThingId;
+
+/**
+ * Thrown if historical data of the Thing was either not present in Ditto at all or if the requester had insufficient
+ * permissions to access it.
+ *
+ * @since 3.2.0
+ */
+@Immutable
+@JsonParsableException(errorCode = ThingHistoryNotAccessibleException.ERROR_CODE)
+public final class ThingHistoryNotAccessibleException extends DittoRuntimeException implements ThingException {
+
+    /**
+     * Error code of this exception.
+     */
+    public static final String ERROR_CODE = ERROR_CODE_PREFIX + "thing.history.notfound";
+
+    private static final String MESSAGE_TEMPLATE =
+            "The Thing with ID ''{0}'' at revision ''{1}'' could not be found or requester had insufficient " +
+                    "permissions to access it.";
+
+    private static final String MESSAGE_TEMPLATE_TS =
+            "The Thing with ID ''{0}'' at timestamp ''{1}'' could not be found or requester had insufficient " +
+                    "permissions to access it.";
+
+    private static final String DEFAULT_DESCRIPTION =
+            "Check if the ID of your requested Thing was correct, you have sufficient permissions and ensure that the " +
+                    "asked for revision/timestamp does not exceed the history-retention-duration.";
+
+    private static final long serialVersionUID = 8883736111094383234L;
+
+    private ThingHistoryNotAccessibleException(final DittoHeaders dittoHeaders,
+            @Nullable final String message,
+            @Nullable final String description,
+            @Nullable final Throwable cause,
+            @Nullable final URI href) {
+        super(ERROR_CODE, HttpStatus.NOT_FOUND, dittoHeaders, message, description, cause, href);
+    }
+
+    private static String getMessage(final ThingId thingId, final long revision) {
+        checkNotNull(thingId, "thingId");
+        return MessageFormat.format(MESSAGE_TEMPLATE, String.valueOf(thingId), String.valueOf(revision));
+    }
+
+    private static String getMessage(final ThingId thingId, final Instant timestamp) {
+        checkNotNull(thingId, "thingId");
+        checkNotNull(timestamp, "timestamp");
+        return MessageFormat.format(MESSAGE_TEMPLATE_TS, String.valueOf(thingId), timestamp.toString());
+    }
+
+    /**
+     * A mutable builder for a {@code ThingHistoryNotAccessibleException}.
+     *
+     * @param thingId the ID of the thing.
+     * @param revision the asked for revision of the thing.
+     * @return the builder.
+     * @throws NullPointerException if {@code thingId} is {@code null}.
+     */
+    public static Builder newBuilder(final ThingId thingId, final long revision) {
+        return new Builder(thingId, revision);
+    }
+
+    /**
+     * A mutable builder for a {@code ThingHistoryNotAccessibleException}.
+     *
+     * @param thingId the ID of the thing.
+     * @param timestamp the asked for timestamp of the thing.
+     * @return the builder.
+     * @throws NullPointerException if {@code thingId} is {@code null}.
+     */
+    public static Builder newBuilder(final ThingId thingId, final Instant timestamp) {
+        return new Builder(thingId, timestamp);
+    }
+
+    /**
+     * Constructs a new {@code ThingHistoryNotAccessibleException} object with given message.
+     *
+     * @param message detail message. This message can be later retrieved by the {@link #getMessage()} method.
+     * @param dittoHeaders the headers of the command which resulted in this exception.
+     * @return the new ThingHistoryNotAccessibleException.
+     * @throws NullPointerException if {@code dittoHeaders} is {@code null}.
+     */
+    public static ThingHistoryNotAccessibleException fromMessage(@Nullable final String message,
+            final DittoHeaders dittoHeaders) {
+        return DittoRuntimeException.fromMessage(message, dittoHeaders, new Builder());
+    }
+
+    /**
+     * Constructs a new {@code ThingHistoryNotAccessibleException} object with the exception message extracted from the given
+     * JSON object.
+     *
+     * @param jsonObject the JSON to read the {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from.
+     * @param dittoHeaders the headers of the command which resulted in this exception.
+     * @return the new ThingHistoryNotAccessibleException.
+     * @throws NullPointerException if any argument is {@code null}.
+     * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message.
+     * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected
+     * format.
+     */
+    public static ThingHistoryNotAccessibleException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) {
+        return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder());
+    }
+
+    @Override
+    public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) {
+        return new Builder()
+                .message(getMessage())
+                .description(getDescription().orElse(null))
+                .cause(getCause())
+                .href(getHref().orElse(null))
+                .dittoHeaders(dittoHeaders)
+                .build();
+    }
+
+    /**
+     * A mutable builder with a fluent API for a {@link org.eclipse.ditto.things.model.signals.commands.exceptions.ThingHistoryNotAccessibleException}.
+     */
+    @NotThreadSafe
+    public static final class Builder extends DittoRuntimeExceptionBuilder {
+
+        private Builder() {
+            description(DEFAULT_DESCRIPTION);
+        }
+
+        private Builder(final ThingId thingId, final long revision) {
+            this();
+            message(ThingHistoryNotAccessibleException.getMessage(thingId, revision));
+        }
+
+        private Builder(final ThingId thingId, final Instant timestamp) {
+            this();
+            message(ThingHistoryNotAccessibleException.getMessage(thingId, timestamp));
+        }
+
+        @Override
+        protected ThingHistoryNotAccessibleException doBuild(final DittoHeaders dittoHeaders,
+                @Nullable final String message,
+                @Nullable final String description,
+                @Nullable final Throwable cause,
+                @Nullable final URI href) {
+            return new ThingHistoryNotAccessibleException(dittoHeaders, message, description, cause, href);
+        }
+
+    }
+
+}
diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingPreconditionNotModifiedException.java b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingPreconditionNotModifiedException.java
index 178a5b3c6b4..f5a0c093d04 100644
--- a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingPreconditionNotModifiedException.java
+++ b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingPreconditionNotModifiedException.java
@@ -19,12 +19,12 @@
 import javax.annotation.concurrent.Immutable;
 import javax.annotation.concurrent.NotThreadSafe;
 
-import org.eclipse.ditto.json.JsonObject;
 import org.eclipse.ditto.base.model.common.HttpStatus;
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException;
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.base.model.json.JsonParsableException;
+import org.eclipse.ditto.json.JsonObject;
 import org.eclipse.ditto.things.model.ThingException;
 
 
@@ -57,6 +57,16 @@ private ThingPreconditionNotModifiedException(final DittoHeaders dittoHeaders,
         super(ERROR_CODE, HttpStatus.NOT_MODIFIED, dittoHeaders, message, description, cause, href);
     }
 
+    /**
+     * A mutable builder for a {@link ThingPreconditionNotModifiedException}.
+     *
+     * @return the builder.
+     * @since 3.3.0
+     */
+    public static Builder newBuilder() {
+        return new Builder();
+    }
+
     /**
      * A mutable builder for a {@link ThingPreconditionNotModifiedException}.
      *
diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/MergeThing.java b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/MergeThing.java
index c64d31644da..eccbfde79e5 100644
--- a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/MergeThing.java
+++ b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/MergeThing.java
@@ -21,6 +21,7 @@
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
 
+import org.eclipse.ditto.base.model.common.Placeholders;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.base.model.json.FieldType;
 import org.eclipse.ditto.base.model.json.JsonParsableCommand;
@@ -36,6 +37,7 @@
 import org.eclipse.ditto.json.JsonObjectBuilder;
 import org.eclipse.ditto.json.JsonPointer;
 import org.eclipse.ditto.json.JsonValue;
+import org.eclipse.ditto.policies.model.Policy;
 import org.eclipse.ditto.policies.model.PolicyId;
 import org.eclipse.ditto.things.model.Attributes;
 import org.eclipse.ditto.things.model.AttributesModelFactory;
@@ -50,6 +52,7 @@
 import org.eclipse.ditto.things.model.signals.commands.ThingCommand;
 import org.eclipse.ditto.things.model.signals.commands.ThingCommandSizeValidator;
 import org.eclipse.ditto.things.model.signals.commands.exceptions.AttributePointerInvalidException;
+import org.eclipse.ditto.things.model.signals.commands.exceptions.PoliciesConflictingException;
 import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingIdNotExplicitlySettableException;
 import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingMergeInvalidException;
 
@@ -75,16 +78,51 @@ public final class MergeThing extends AbstractCommand implements Thi
      */
     public static final String TYPE = TYPE_PREFIX + NAME;
 
+    /**
+     * Json Field definition for the optional initial "inline" policy for usage in getEntity().
+     */
+    public static final JsonFieldDefinition JSON_INLINE_POLICY =
+            JsonFactory.newJsonObjectFieldDefinition(Policy.INLINED_FIELD_NAME, FieldType.REGULAR, JsonSchemaVersion.V_2);
+
+    /**
+     * Json Field definition for the optional feature to copy an existing policy.
+     */
+    public static final JsonFieldDefinition JSON_COPY_POLICY_FROM =
+            JsonFactory.newStringFieldDefinition("_copyPolicyFrom", FieldType.REGULAR, JsonSchemaVersion.V_2);
+
+    private static final JsonFieldDefinition JSON_INITIAL_POLICY =
+            JsonFactory.newJsonObjectFieldDefinition("initialPolicy", FieldType.REGULAR, JsonSchemaVersion.V_2);
+
+    private static final JsonFieldDefinition JSON_POLICY_ID_OR_PLACEHOLDER =
+            JsonFactory.newStringFieldDefinition("policyIdOrPlaceholder", FieldType.REGULAR, JsonSchemaVersion.V_2);
+
     private final ThingId thingId;
     private final JsonPointer path;
     private final JsonValue value;
 
+    @Nullable private final JsonObject initialPolicy;
+    @Nullable private final String policyIdOrPlaceholder;
+
     private MergeThing(final ThingId thingId, final JsonPointer path, final JsonValue value,
+            @Nullable final JsonObject initialPolicy,
+            @Nullable final String policyIdOrPlaceholder,
             final DittoHeaders dittoHeaders) {
         super(TYPE, FeatureToggle.checkMergeFeatureEnabled(TYPE, dittoHeaders));
         this.thingId = checkNotNull(thingId, "thingId");
         this.path = checkNotNull(path, "path");
         this.value = checkJsonSize(checkNotNull(value, "value"), dittoHeaders);
+
+        if (policyIdOrPlaceholder != null && initialPolicy != null) {
+            throw PoliciesConflictingException.newBuilder(thingId).dittoHeaders(dittoHeaders).build();
+        }
+
+        if (policyIdOrPlaceholder != null && !Placeholders.containsAnyPlaceholder(policyIdOrPlaceholder)) {
+            PolicyId.of(policyIdOrPlaceholder); //validates
+        }
+
+        this.initialPolicy = initialPolicy;
+        this.policyIdOrPlaceholder = policyIdOrPlaceholder;
+
         checkSchemaVersion();
     }
 
@@ -100,7 +138,15 @@ private MergeThing(final ThingId thingId, final JsonPointer path, final JsonValu
      */
     public static MergeThing of(final ThingId thingId, final JsonPointer path, final JsonValue value,
             final DittoHeaders dittoHeaders) {
-        return new MergeThing(thingId, path, value, dittoHeaders);
+        if (path.isEmpty() && value.isObject()) {
+            final JsonObject object = value.asObject();
+            final Optional initialPolicy = initialPolicyForMergeThingFrom(object);
+            final Optional policyIdOrPlaceholder = policyIdOrPlaceholderForMergeThingFrom(object);
+            return new MergeThing(thingId, path, value, initialPolicy.orElse(null),
+                    policyIdOrPlaceholder.orElse(null), dittoHeaders);
+        } else {
+            return new MergeThing(thingId, path, value, null, null, dittoHeaders);
+        }
     }
 
     /**
@@ -115,7 +161,32 @@ public static MergeThing withThing(final ThingId thingId, final Thing thing, fin
         ensureThingIdMatches(thingId, thing);
         ensureThingIsNotNullOrEmpty(thing, dittoHeaders);
         final JsonObject mergePatch = thing.toJson();
-        return new MergeThing(thingId, JsonPointer.empty(), mergePatch, dittoHeaders);
+        return new MergeThing(thingId, JsonPointer.empty(), mergePatch, null, null, dittoHeaders);
+    }
+
+    /**
+     * Creates a command for merging the thing identified by {@code thingId} with the given {@code thing}.
+     * Only one of the arguments initialPolicy and policyIdOrPlaceholder must not be null. They are both allowed to be
+     * null, but not both to not be null at the same time.
+     *
+     * @param thingId the thing id.
+     * @param thing the thing that is merged with the existing thing.
+     * @param policyIdOrPlaceholder the policy id of the {@code Policy} to copy and set for the Thing when creating it.
+     * If it's a placeholder it will be resolved to a policy id.
+     * Placeholder must be of the syntax: {@code {{ ref:things//policyId }} }.
+     * @param initialPolicy the initial {@code Policy} to set for the Thing when creating it - may be null.
+     * @param dittoHeaders the ditto headers.
+     * @return the created {@link MergeThing} command.
+     * @since 3.3.0
+     */
+    public static MergeThing withThing(final ThingId thingId, final Thing thing,
+            @Nullable final JsonObject initialPolicy, @Nullable final String policyIdOrPlaceholder,
+            final DittoHeaders dittoHeaders) {
+        ensureThingIdMatches(thingId, thing);
+        ensureThingIsNotNullOrEmpty(thing, dittoHeaders);
+        final JsonObject mergePatch = thing.toJson();
+        return new MergeThing(thingId, JsonPointer.empty(), mergePatch, initialPolicy, policyIdOrPlaceholder,
+                dittoHeaders);
     }
 
     /**
@@ -129,7 +200,8 @@ public static MergeThing withThing(final ThingId thingId, final Thing thing, fin
     public static MergeThing withPolicyId(final ThingId thingId, final PolicyId policyId,
             final DittoHeaders dittoHeaders) {
         checkNotNull(policyId, "policyId");
-        return new MergeThing(thingId, Thing.JsonFields.POLICY_ID.getPointer(), JsonValue.of(policyId), dittoHeaders);
+        return new MergeThing(thingId, Thing.JsonFields.POLICY_ID.getPointer(), JsonValue.of(policyId),
+                null, null, dittoHeaders);
     }
 
     /**
@@ -143,7 +215,7 @@ public static MergeThing withPolicyId(final ThingId thingId, final PolicyId poli
     public static MergeThing withThingDefinition(final ThingId thingId, final ThingDefinition thingDefinition,
             final DittoHeaders dittoHeaders) {
         return new MergeThing(thingId, Thing.JsonFields.DEFINITION.getPointer(), thingDefinition.toJson(),
-                dittoHeaders);
+                null, null, dittoHeaders);
     }
 
     /**
@@ -156,7 +228,8 @@ public static MergeThing withThingDefinition(final ThingId thingId, final ThingD
      */
     public static MergeThing withAttributes(final ThingId thingId, final Attributes attributes,
             final DittoHeaders dittoHeaders) {
-        return new MergeThing(thingId, Thing.JsonFields.ATTRIBUTES.getPointer(), attributes.toJson(), dittoHeaders);
+        return new MergeThing(thingId, Thing.JsonFields.ATTRIBUTES.getPointer(), attributes.toJson(),
+                null, null, dittoHeaders);
     }
 
     /**
@@ -173,7 +246,7 @@ public static MergeThing withAttribute(final ThingId thingId, final JsonPointer
             final JsonValue attributeValue, final DittoHeaders dittoHeaders) {
         final JsonPointer absolutePath =
                 Thing.JsonFields.ATTRIBUTES.getPointer().append(checkAttributePointer(attributePath, dittoHeaders));
-        return new MergeThing(thingId, absolutePath, checkAttributeValue(attributeValue), dittoHeaders);
+        return new MergeThing(thingId, absolutePath, checkAttributeValue(attributeValue), null, null, dittoHeaders);
     }
 
     /**
@@ -187,7 +260,7 @@ public static MergeThing withAttribute(final ThingId thingId, final JsonPointer
     public static MergeThing withFeatures(final ThingId thingId, final Features features,
             final DittoHeaders dittoHeaders) {
         final JsonPointer absolutePath = Thing.JsonFields.FEATURES.getPointer();
-        return new MergeThing(thingId, absolutePath, features.toJson(), dittoHeaders);
+        return new MergeThing(thingId, absolutePath, features.toJson(), null, null, dittoHeaders);
     }
 
     /**
@@ -201,7 +274,7 @@ public static MergeThing withFeatures(final ThingId thingId, final Features feat
     public static MergeThing withFeature(final ThingId thingId, final Feature feature,
             final DittoHeaders dittoHeaders) {
         final JsonPointer absolutePath = Thing.JsonFields.FEATURES.getPointer().append(JsonPointer.of(feature.getId()));
-        return new MergeThing(thingId, absolutePath, feature.toJson(), dittoHeaders);
+        return new MergeThing(thingId, absolutePath, feature.toJson(), null, null, dittoHeaders);
     }
 
     /**
@@ -220,7 +293,7 @@ public static MergeThing withFeatureDefinition(final ThingId thingId,
         final JsonPointer absolutePath = Thing.JsonFields.FEATURES.getPointer()
                 .append(JsonPointer.of(featureId))
                 .append(Feature.JsonFields.DEFINITION.getPointer());
-        return new MergeThing(thingId, absolutePath, featureDefinition.toJson(), dittoHeaders);
+        return new MergeThing(thingId, absolutePath, featureDefinition.toJson(), null, null, dittoHeaders);
     }
 
     /**
@@ -239,7 +312,7 @@ public static MergeThing withFeatureProperties(final ThingId thingId,
         final JsonPointer absolutePath = Thing.JsonFields.FEATURES.getPointer()
                 .append(JsonPointer.of(featureId))
                 .append(Feature.JsonFields.PROPERTIES.getPointer());
-        return new MergeThing(thingId, absolutePath, featureProperties.toJson(), dittoHeaders);
+        return new MergeThing(thingId, absolutePath, featureProperties.toJson(), null, null, dittoHeaders);
     }
 
     /**
@@ -260,7 +333,7 @@ public static MergeThing withFeatureProperty(final ThingId thingId,
                 .append(JsonPointer.of(featureId))
                 .append(Feature.JsonFields.PROPERTIES.getPointer())
                 .append(checkPropertyPointer(propertyPath));
-        return new MergeThing(thingId, absolutePath, checkPropertyValue(propertyValue), dittoHeaders);
+        return new MergeThing(thingId, absolutePath, checkPropertyValue(propertyValue), null, null, dittoHeaders);
     }
 
     /**
@@ -279,7 +352,7 @@ public static MergeThing withFeatureDesiredProperties(final ThingId thingId,
         final JsonPointer absolutePath = Thing.JsonFields.FEATURES.getPointer()
                 .append(JsonPointer.of(featureId))
                 .append(Feature.JsonFields.DESIRED_PROPERTIES.getPointer());
-        return new MergeThing(thingId, absolutePath, desiredFeatureProperties.toJson(), dittoHeaders);
+        return new MergeThing(thingId, absolutePath, desiredFeatureProperties.toJson(), null, null, dittoHeaders);
     }
 
     /**
@@ -300,7 +373,30 @@ public static MergeThing withFeatureDesiredProperty(final ThingId thingId,
                 .append(JsonPointer.of(featureId))
                 .append(Feature.JsonFields.DESIRED_PROPERTIES.getPointer())
                 .append(checkPropertyPointer(propertyPath));
-        return new MergeThing(thingId, absolutePath, checkPropertyValue(propertyValue), dittoHeaders);
+        return new MergeThing(thingId, absolutePath, checkPropertyValue(propertyValue), null, null, dittoHeaders);
+    }
+
+    /**
+     * Retrieves a potentially included "inline policy" from the {@link #JSON_INLINE_POLICY _policy} field of the passed
+     * {@code jsonObject}.
+     *
+     * @param jsonObject the JSON object to look for an inline policy in.
+     * @return the potentially contained inline policy as JSON object.
+     */
+    public static Optional initialPolicyForMergeThingFrom(final JsonObject jsonObject) {
+        return jsonObject.getValue(JSON_INLINE_POLICY)
+                .map(JsonValue::asObject);
+    }
+
+    /**
+     * Retrieves a potentially included "policy id or placeholder" to copy a policy from the
+     * {@link #JSON_COPY_POLICY_FROM _copyPolicyFrom} field of the passed {@code jsonObject}.
+     *
+     * @param jsonObject the JSON object to look for the policy id or placeholder in.
+     * @return the potentially contained policy id or placeholder.
+     */
+    public static Optional policyIdOrPlaceholderForMergeThingFrom(final JsonObject jsonObject) {
+        return jsonObject.getValue(JSON_COPY_POLICY_FROM);
     }
 
     private static JsonPointer checkPropertyPointer(final JsonPointer propertyPointer) {
@@ -379,8 +475,11 @@ public static MergeThing fromJson(final JsonObject jsonObject, final DittoHeader
             final String thingId = jsonObject.getValueOrThrow(ThingCommand.JsonFields.JSON_THING_ID);
             final String path = jsonObject.getValueOrThrow(JsonFields.JSON_PATH);
             final JsonValue jsonValue = jsonObject.getValueOrThrow(JsonFields.JSON_VALUE);
+            final JsonObject initialPolicyObject = jsonObject.getValue(JSON_INITIAL_POLICY).orElse(null);
+            final String policyIdOrPlaceholder = jsonObject.getValue(JSON_POLICY_ID_OR_PLACEHOLDER).orElse(null);
 
-            return of(ThingId.of(thingId), JsonPointer.of(path), jsonValue, dittoHeaders);
+            return new MergeThing(ThingId.of(thingId), JsonPointer.of(path), jsonValue, initialPolicyObject,
+                    policyIdOrPlaceholder, dittoHeaders);
         });
     }
 
@@ -398,12 +497,22 @@ public ThingId getEntityId() {
 
     @Override
     public Optional getEntity() {
-        return Optional.of(value);
+        if (path.isEmpty()) {
+            final JsonObject thingJson = value.asObject();
+            final JsonObject withInlinePolicyThingJson =
+                    getInitialPolicy().map(ip -> thingJson.set(JSON_INLINE_POLICY, ip)).orElse(thingJson);
+            final JsonObject fullThingJson = getPolicyIdOrPlaceholder().map(
+                    containedPolicyIdOrPlaceholder -> withInlinePolicyThingJson.set(JSON_COPY_POLICY_FROM,
+                            containedPolicyIdOrPlaceholder)).orElse(withInlinePolicyThingJson);
+            return Optional.of(fullThingJson);
+        } else {
+            return Optional.of(value);
+        }
     }
 
     @Override
     public Optional getEntity(final JsonSchemaVersion schemaVersion) {
-        return Optional.of(value);
+        return getEntity();
     }
 
     @Override
@@ -411,6 +520,23 @@ public JsonPointer getResourcePath() {
         return path;
     }
 
+    /**
+     * @return the initial {@code Policy} if there should be one applied when creating the Thing.
+     * @since 3.3.0
+     */
+    public Optional getInitialPolicy() {
+        return Optional.ofNullable(initialPolicy);
+    }
+
+    /**
+     * @return The policy id of the {@code Policy} to copy and set for the Thing when creating it.
+     * Could also be a placeholder like: {{ ref:things/theThingId/policyId }}.
+     * @since 3.3.0
+     */
+    public Optional getPolicyIdOrPlaceholder() {
+        return Optional.ofNullable(policyIdOrPlaceholder);
+    }
+
     @Override
     public boolean changesAuthorization() {
         return Thing.JsonFields.POLICY_ID.getPointer().equals(path) || path.isEmpty() && value.isObject() &&
@@ -419,12 +545,12 @@ public boolean changesAuthorization() {
 
     @Override
     public Category getCategory() {
-        return Category.MODIFY;
+        return Category.MERGE;
     }
 
     @Override
     public MergeThing setDittoHeaders(final DittoHeaders dittoHeaders) {
-        return of(thingId, path, value, dittoHeaders);
+        return new MergeThing(thingId, path, value, initialPolicy, policyIdOrPlaceholder, dittoHeaders);
     }
 
     @Override
@@ -434,6 +560,12 @@ protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final Js
         jsonObjectBuilder.set(ThingCommand.JsonFields.JSON_THING_ID, thingId.toString(), predicate);
         jsonObjectBuilder.set(JsonFields.JSON_PATH, path.toString(), predicate);
         jsonObjectBuilder.set(JsonFields.JSON_VALUE, value, predicate);
+        if (initialPolicy != null) {
+            jsonObjectBuilder.set(JSON_INITIAL_POLICY, initialPolicy, predicate);
+        }
+        if (policyIdOrPlaceholder != null) {
+            jsonObjectBuilder.set(JSON_POLICY_ID_OR_PLACEHOLDER, policyIdOrPlaceholder, predicate);
+        }
     }
 
     @Override
@@ -484,8 +616,12 @@ public boolean equals(final Object o) {
             return false;
         }
         final MergeThing that = (MergeThing) o;
-        return that.canEqual(this) && thingId.equals(that.thingId) && path.equals(that.path) &&
-                value.equals(that.value);
+        return that.canEqual(this) &&
+                Objects.equals(thingId, that.thingId) &&
+                Objects.equals(path, that.path) &&
+                Objects.equals(value, that.value) &&
+                Objects.equals(initialPolicy, that.initialPolicy) &&
+                Objects.equals(policyIdOrPlaceholder, that.policyIdOrPlaceholder);
     }
 
     @Override
@@ -495,7 +631,7 @@ protected boolean canEqual(@Nullable final Object other) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), thingId, path, value);
+        return Objects.hash(super.hashCode(), thingId, path, value, initialPolicy, policyIdOrPlaceholder);
     }
 
     @Override
@@ -505,6 +641,8 @@ public String toString() {
                 ", thingId=" + thingId +
                 ", path=" + path +
                 ", value=" + value +
+                ", initialPolicy=" + initialPolicy +
+                ", policyIdOrPlaceholder=" + policyIdOrPlaceholder +
                 "]";
     }
 }
diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/ModifyThing.java b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/ModifyThing.java
index 466287d1d87..0b40da9b8df 100755
--- a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/ModifyThing.java
+++ b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/ModifyThing.java
@@ -35,6 +35,7 @@
 import org.eclipse.ditto.json.JsonObjectBuilder;
 import org.eclipse.ditto.json.JsonPointer;
 import org.eclipse.ditto.json.JsonValue;
+import org.eclipse.ditto.policies.model.Policy;
 import org.eclipse.ditto.policies.model.PolicyId;
 import org.eclipse.ditto.things.model.Thing;
 import org.eclipse.ditto.things.model.ThingId;
@@ -86,7 +87,7 @@ public final class ModifyThing extends AbstractCommand implements T
      * Json Field definition for the optional initial "inline" policy for usage in getEntity().
      */
     public static final JsonFieldDefinition JSON_INLINE_POLICY =
-            JsonFactory.newJsonObjectFieldDefinition("_policy", FieldType.REGULAR, JsonSchemaVersion.V_2);
+            JsonFactory.newJsonObjectFieldDefinition(Policy.INLINED_FIELD_NAME, FieldType.REGULAR, JsonSchemaVersion.V_2);
 
     private final ThingId thingId;
     private final Thing thing;
diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/events/ThingEventToThingConverter.java b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/events/ThingEventToThingConverter.java
index 7c3e6099fec..0b63e237694 100644
--- a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/events/ThingEventToThingConverter.java
+++ b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/events/ThingEventToThingConverter.java
@@ -175,6 +175,25 @@ private static Map, BiFunction, ThingBuilder.FromScratch,
                         ((FeaturePropertyModified) te).getPropertyValue()).build());
         mappers.put(FeaturePropertyDeleted.class, (te, tb) -> tb.build());
 
+        mappers.put(FeatureDesiredPropertiesCreated.class, (te, tb) -> tb.setFeature(Feature.newBuilder()
+                .desiredProperties(((FeatureDesiredPropertiesCreated) te).getDesiredProperties())
+                .withId(((FeatureDesiredPropertiesCreated) te).getFeatureId())
+                .build()).build());
+        mappers.put(FeatureDesiredPropertiesModified.class, (te, tb) -> tb.setFeature(Feature.newBuilder()
+                .desiredProperties(((FeatureDesiredPropertiesModified) te).getDesiredProperties())
+                .withId(((FeatureDesiredPropertiesModified) te).getFeatureId())
+                .build()).build());
+        mappers.put(FeatureDesiredPropertiesDeleted.class, (te, tb) -> tb.build());
+        mappers.put(FeatureDesiredPropertyCreated.class, (te, tb) ->
+                tb.setFeatureDesiredProperty(((FeatureDesiredPropertyCreated) te).getFeatureId(),
+                        ((FeatureDesiredPropertyCreated) te).getDesiredPropertyPointer(),
+                        ((FeatureDesiredPropertyCreated) te).getDesiredPropertyValue()).build());
+        mappers.put(FeatureDesiredPropertyModified.class, (te, tb) ->
+                tb.setFeatureDesiredProperty(((FeatureDesiredPropertyModified) te).getFeatureId(),
+                        ((FeatureDesiredPropertyModified) te).getDesiredPropertyPointer(),
+                        ((FeatureDesiredPropertyModified) te).getDesiredPropertyValue()).build());
+        mappers.put(FeatureDesiredPropertyDeleted.class, (te, tb) -> tb.build());
+
         return mappers;
     }
 
diff --git a/things/model/src/test/java/org/eclipse/ditto/things/model/signals/commands/modify/MergeThingTest.java b/things/model/src/test/java/org/eclipse/ditto/things/model/signals/commands/modify/MergeThingTest.java
index dedbb9e36c5..d2d57300bb9 100644
--- a/things/model/src/test/java/org/eclipse/ditto/things/model/signals/commands/modify/MergeThingTest.java
+++ b/things/model/src/test/java/org/eclipse/ditto/things/model/signals/commands/modify/MergeThingTest.java
@@ -18,10 +18,12 @@
 import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf;
 import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable;
 
+import java.text.MessageFormat;
 import java.util.Optional;
 import java.util.UUID;
 
 import org.assertj.core.api.Assertions;
+import org.assertj.core.api.JUnitSoftAssertions;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.json.JsonFactory;
 import org.eclipse.ditto.json.JsonKeyInvalidException;
@@ -37,6 +39,8 @@
 import org.eclipse.ditto.things.model.ThingsModelFactory;
 import org.eclipse.ditto.things.model.signals.commands.TestConstants;
 import org.eclipse.ditto.things.model.signals.commands.ThingCommand;
+import org.eclipse.ditto.things.model.signals.commands.exceptions.PoliciesConflictingException;
+import org.junit.Rule;
 import org.junit.Test;
 
 import nl.jqno.equalsverifier.EqualsVerifier;
@@ -58,11 +62,14 @@ public final class MergeThingTest {
     private static final MergeThing KNOWN_MERGE_THING = MergeThing.withAttribute(TestConstants.Thing.THING_ID,
             TestConstants.Thing.LOCATION_ATTRIBUTE_POINTER, TestConstants.Thing.LOCATION_ATTRIBUTE_VALUE, DITTO_HEADERS);
 
+    @Rule
+    public final JUnitSoftAssertions softly = new JUnitSoftAssertions();
+
     @Test
     public void assertImmutability() {
         assertInstancesOf(MergeThing.class,
                 areImmutable(),
-                provided(JsonPointer.class, JsonValue.class, ThingId.class).isAlsoImmutable());
+                provided(JsonPointer.class, JsonValue.class, ThingId.class, JsonObject.class).isAlsoImmutable());
     }
 
     @Test
@@ -426,4 +433,51 @@ public void mergeTooLargeThing() {
                 .isInstanceOf(ThingTooLargeException.class);
     }
 
+    @Test
+    public void initializeWithInitialPolicyNullAndWithCopiedPolicyNull() {
+        final MergeThing modifyThing =
+                MergeThing.withThing(TestConstants.Thing.THING_ID, TestConstants.Thing.THING, null, null,
+                        TestConstants.EMPTY_DITTO_HEADERS);
+
+        softly.assertThat(modifyThing.getInitialPolicy()).isNotPresent();
+        softly.assertThat(modifyThing.getPolicyIdOrPlaceholder()).isNotPresent();
+    }
+
+    @Test
+    public void initializeWithCopiedPolicy() {
+        final String thingReference = "{{ ref:things/my_namespace:my_thing/policyId }}";
+        final MergeThing modifyThing =
+                MergeThing.withThing(TestConstants.Thing.THING_ID, TestConstants.Thing.THING, null, thingReference,
+                        TestConstants.EMPTY_DITTO_HEADERS);
+
+        softly.assertThat(modifyThing.getInitialPolicy()).isNotPresent();
+        softly.assertThat(modifyThing.getPolicyIdOrPlaceholder()).isPresent();
+        softly.assertThat(modifyThing.getPolicyIdOrPlaceholder()).contains(thingReference);
+    }
+
+    @Test
+    public void initializeWithCopiedPolicyAndWithInitialPolicyNullAndPolicyIdNull() {
+        final Thing thing = TestConstants.Thing.THING.setPolicyId(null);
+        final String thingReference = "{{ ref:things/my_namespace:my_thing/policyId }}";
+        final MergeThing modifyThing =
+                MergeThing.withThing(TestConstants.Thing.THING_ID, thing, null, thingReference,
+                        TestConstants.EMPTY_DITTO_HEADERS);
+
+        softly.assertThat(modifyThing.getInitialPolicy()).isNotPresent();
+        softly.assertThat(modifyThing.getPolicyIdOrPlaceholder()).isPresent();
+        softly.assertThat(modifyThing.getPolicyIdOrPlaceholder()).contains(thingReference);
+    }
+
+    @Test
+    public void initializeWithCopiedPolicyAndWithInitialPolicy() {
+        final String thingReference = "{{ ref:things/my_namespace:my_thing/policyId }}";
+        softly.assertThatThrownBy(() ->
+                MergeThing.withThing(TestConstants.Thing.THING_ID, TestConstants.Thing.THING,
+                                JsonObject.newBuilder().build(), thingReference, TestConstants.EMPTY_DITTO_HEADERS)
+                )
+                .isInstanceOf(PoliciesConflictingException.class)
+                .hasMessage(MessageFormat.format(
+                        "The Thing with ID ''{0}'' could not be created as it contained an inline Policy as" +
+                                " well as a policyID to copy.", TestConstants.Thing.THING_ID));
+    }
 }
diff --git a/things/model/src/test/java/org/eclipse/ditto/things/model/signals/commands/query/RetrieveThingsTest.java b/things/model/src/test/java/org/eclipse/ditto/things/model/signals/commands/query/RetrieveThingsTest.java
index 2c42c167df4..a3b2668a3b0 100755
--- a/things/model/src/test/java/org/eclipse/ditto/things/model/signals/commands/query/RetrieveThingsTest.java
+++ b/things/model/src/test/java/org/eclipse/ditto/things/model/signals/commands/query/RetrieveThingsTest.java
@@ -183,7 +183,7 @@ public void createInstanceWithInvalidNamespacesThrowsException() {
 
     @Test
     public void checkRetrieveThingsWithEmptyJsonFieldSelectorBehavesEquallyAsOmittingFields() {
-        final JsonFieldSelector selectedFields = JsonFactory.newFieldSelector(null, JSON_PARSE_OPTIONS);
+        final JsonFieldSelector selectedFields = JsonFactory.newFieldSelector((String) null, JSON_PARSE_OPTIONS);
         final RetrieveThings retrieveThings = RetrieveThings
                 .getBuilder(TestConstants.Thing.THING_ID, ThingId.inDefaultNamespace("AnotherThingId"))
                 .selectedFields(selectedFields)
diff --git a/things/model/src/test/java/org/eclipse/ditto/things/model/signals/events/ThingEventToThingConverterTest.java b/things/model/src/test/java/org/eclipse/ditto/things/model/signals/events/ThingEventToThingConverterTest.java
index 96331f28955..aa58be822fa 100644
--- a/things/model/src/test/java/org/eclipse/ditto/things/model/signals/events/ThingEventToThingConverterTest.java
+++ b/things/model/src/test/java/org/eclipse/ditto/things/model/signals/events/ThingEventToThingConverterTest.java
@@ -21,11 +21,7 @@
 import org.eclipse.ditto.json.JsonObject;
 import org.eclipse.ditto.json.JsonPointer;
 import org.eclipse.ditto.json.JsonValue;
-import org.eclipse.ditto.things.model.Attributes;
-import org.eclipse.ditto.things.model.Feature;
-import org.eclipse.ditto.things.model.FeatureProperties;
-import org.eclipse.ditto.things.model.Thing;
-import org.eclipse.ditto.things.model.ThingRevision;
+import org.eclipse.ditto.things.model.*;
 import org.junit.Test;
 
 /**
@@ -39,6 +35,8 @@ public final class ThingEventToThingConverterTest {
     private static final String ATTR_KEY_CITY = "city";
     private static final String KNOWN_CITY = "Immenstaad am Bodensee";
 
+    private static final JsonPointer DESIRED_PROPERTY_POINTER = JsonPointer.of("target_year_1");
+
     @Test
     public void ensureMergeWithExtraFieldsMergesCorrectly() {
         final long revision = 23L;
@@ -124,4 +122,104 @@ public void ensureThingMergedConvertsAsExpected() {
         assertThat(thing.getRevision()).contains(ThingRevision.newInstance(revision));
         assertThat(thing.getAttributes()).contains(Attributes.newBuilder().set("foo", "bar").build());
     }
+
+    @Test
+    public void testThingEventToThingConvertsFeatureDesiredPropertiesCreated() {
+        long revision = 23L;
+        FeatureDesiredPropertiesCreated desiredPropertiesCreated = FeatureDesiredPropertiesCreated.of(
+                TestConstants.Thing.THING_ID, TestConstants.Feature.FLUX_CAPACITOR_ID,
+                TestConstants.Feature.FLUX_CAPACITOR_PROPERTIES, revision,null, DittoHeaders.empty(), null);
+
+        Optional thingOpt = ThingEventToThingConverter.thingEventToThing(desiredPropertiesCreated);
+        assertThat(thingOpt).isPresent();
+        Thing thing = thingOpt.get();
+        assertThat(thing.getEntityId()).contains(TestConstants.Thing.THING_ID);
+        assertThat(thing.getRevision()).contains(ThingRevision.newInstance(revision));
+        FeatureProperties desiredProperties = getDesiredProperties(thing);
+        assertThat(desiredProperties).isEqualTo(TestConstants.Feature.FLUX_CAPACITOR_PROPERTIES);
+    }
+
+    private FeatureProperties getDesiredProperties(Thing thing) {
+        Optional featuresOpt = thing.getFeatures();
+        assertThat(featuresOpt).isPresent();
+        Optional featureOpt = featuresOpt.get().getFeature(TestConstants.Feature.FLUX_CAPACITOR_ID);
+        assertThat(featureOpt).isPresent();
+        Optional desiredPropertiesOpt = featureOpt.get().getDesiredProperties();
+        assertThat(desiredPropertiesOpt).isPresent();
+        return desiredPropertiesOpt.get();
+    }
+
+    @Test
+    public void testThingEventToThingConvertsFeatureDesiredPropertiesModified() {
+        long revision = 23L;
+        FeatureDesiredPropertiesModified desiredPropertiesModified = FeatureDesiredPropertiesModified.of(
+                TestConstants.Thing.THING_ID, TestConstants.Feature.FLUX_CAPACITOR_ID,
+                TestConstants.Feature.FLUX_CAPACITOR_PROPERTIES, revision,null, DittoHeaders.empty(), null);
+
+        Optional thingOpt = ThingEventToThingConverter.thingEventToThing(desiredPropertiesModified);
+        assertThat(thingOpt).isPresent();
+        Thing thing = thingOpt.get();
+        assertThat(thing.getEntityId()).contains(TestConstants.Thing.THING_ID);
+        assertThat(thing.getRevision()).contains(ThingRevision.newInstance(revision));
+        FeatureProperties desiredProperties = getDesiredProperties(thing);
+        assertThat(desiredProperties).isEqualTo(TestConstants.Feature.FLUX_CAPACITOR_PROPERTIES);
+    }
+
+    @Test
+    public void testThingEventToThingConvertsFeatureDesiredPropertiesDeleted() {
+        long revision = 23L;
+        FeatureDesiredPropertiesDeleted desiredPropertiesDeleted = FeatureDesiredPropertiesDeleted.of(
+                TestConstants.Thing.THING_ID, TestConstants.Feature.FLUX_CAPACITOR_ID, revision,null,
+                DittoHeaders.empty(), null);
+        Optional thingOpt = ThingEventToThingConverter.thingEventToThing(desiredPropertiesDeleted);
+        assertThat(thingOpt).isPresent();
+        Thing thing = thingOpt.get();
+        assertThat(thing.getEntityId()).contains(TestConstants.Thing.THING_ID);
+        assertThat(thing.getRevision()).contains(ThingRevision.newInstance(revision));
+    }
+
+    @Test
+    public void testThingEventToThingConvertsFeatureDesiredPropertyCreated() {
+        long revision = 23L;
+        JsonValue value = JsonValue.of(1955);
+        FeatureDesiredPropertyCreated desiredPropertyCreated = FeatureDesiredPropertyCreated.of(
+                TestConstants.Thing.THING_ID, TestConstants.Feature.FLUX_CAPACITOR_ID, DESIRED_PROPERTY_POINTER, value,
+                revision, null, DittoHeaders.empty(), null);
+        Optional thingOpt = ThingEventToThingConverter.thingEventToThing(desiredPropertyCreated);
+        assertThat(thingOpt).isPresent();
+        Thing thing = thingOpt.get();
+        assertThat(thing.getEntityId()).contains(TestConstants.Thing.THING_ID);
+        assertThat(thing.getRevision()).contains(ThingRevision.newInstance(revision));
+        FeatureProperties desiredProperties = getDesiredProperties(thing);
+        assertThat(desiredProperties).isEqualTo(FeatureProperties.newBuilder().set("target_year_1", 1955).build());
+    }
+
+    @Test
+    public void testThingEventToThingConvertsFeatureDesiredPropertyModified() {
+        long revision = 23L;
+        JsonValue value = JsonValue.of(1955);
+        FeatureDesiredPropertyModified desiredPropertyModified = FeatureDesiredPropertyModified.of(
+                TestConstants.Thing.THING_ID, TestConstants.Feature.FLUX_CAPACITOR_ID, DESIRED_PROPERTY_POINTER, value,
+                revision, null, DittoHeaders.empty(), null);
+        Optional thingOpt = ThingEventToThingConverter.thingEventToThing(desiredPropertyModified);
+        assertThat(thingOpt).isPresent();
+        Thing thing = thingOpt.get();
+        assertThat(thing.getEntityId()).contains(TestConstants.Thing.THING_ID);
+        assertThat(thing.getRevision()).contains(ThingRevision.newInstance(revision));
+        FeatureProperties desiredProperties = getDesiredProperties(thing);
+        assertThat(desiredProperties).isEqualTo(FeatureProperties.newBuilder().set("target_year_1", 1955).build());
+    }
+
+    @Test
+    public void testThingEventToThingConvertsFeatureDesiredPropertyDeleted() {
+        long revision = 23L;
+        FeatureDesiredPropertyDeleted desiredPropertyDeleted = FeatureDesiredPropertyDeleted.of(
+                TestConstants.Thing.THING_ID, TestConstants.Feature.FLUX_CAPACITOR_ID, DESIRED_PROPERTY_POINTER, revision,
+                null, DittoHeaders.empty(), null);
+        Optional thingOpt = ThingEventToThingConverter.thingEventToThing(desiredPropertyDeleted);
+        assertThat(thingOpt).isPresent();
+        Thing thing = thingOpt.get();
+        assertThat(thing.getEntityId()).contains(TestConstants.Thing.THING_ID);
+        assertThat(thing.getRevision()).contains(ThingRevision.newInstance(revision));
+    }
 }
diff --git a/things/service/pom.xml b/things/service/pom.xml
index c0c507bcf15..12119051343 100644
--- a/things/service/pom.xml
+++ b/things/service/pom.xml
@@ -205,12 +205,6 @@
             test
             test-jar
         
-        
-            org.eclipse.ditto
-            ditto-internal-utils-akka
-            test
-            test-jar
-        
 
         
             com.github.docker-java
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/DefaultThingConfig.java b/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/DefaultThingConfig.java
index 00c0774e8f0..649e08d1ca1 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/DefaultThingConfig.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/DefaultThingConfig.java
@@ -23,7 +23,9 @@
 import org.eclipse.ditto.internal.utils.config.ScopedConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultActivityCheckConfig;
+import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultEventConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultSnapshotConfig;
+import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig;
 import org.eclipse.ditto.internal.utils.persistentactors.cleanup.CleanupConfig;
 
@@ -41,6 +43,7 @@ public final class DefaultThingConfig implements ThingConfig {
     private final SupervisorConfig supervisorConfig;
     private final ActivityCheckConfig activityCheckConfig;
     private final SnapshotConfig snapshotConfig;
+    private final EventConfig eventConfig;
     private final CleanupConfig cleanupConfig;
 
     private DefaultThingConfig(final ScopedConfig scopedConfig) {
@@ -48,6 +51,7 @@ private DefaultThingConfig(final ScopedConfig scopedConfig) {
         supervisorConfig = DefaultSupervisorConfig.of(scopedConfig);
         activityCheckConfig = DefaultActivityCheckConfig.of(scopedConfig);
         snapshotConfig = DefaultSnapshotConfig.of(scopedConfig);
+        eventConfig = DefaultEventConfig.of(scopedConfig);
         cleanupConfig = CleanupConfig.of(scopedConfig);
     }
 
@@ -82,6 +86,11 @@ public CleanupConfig getCleanupConfig() {
         return cleanupConfig;
     }
 
+    @Override
+    public EventConfig getEventConfig() {
+        return eventConfig;
+    }
+
     @Override
     public Duration getShutdownTimeout() {
         return shutdownTimeout;
@@ -99,13 +108,15 @@ public boolean equals(final Object o) {
         return Objects.equals(supervisorConfig, that.supervisorConfig) &&
                 Objects.equals(activityCheckConfig, that.activityCheckConfig) &&
                 Objects.equals(snapshotConfig, that.snapshotConfig) &&
+                Objects.equals(eventConfig, that.eventConfig) &&
                 Objects.equals(cleanupConfig, that.cleanupConfig) &&
                 Objects.equals(shutdownTimeout, that.shutdownTimeout);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(supervisorConfig, activityCheckConfig, snapshotConfig, cleanupConfig, shutdownTimeout);
+        return Objects.hash(supervisorConfig, activityCheckConfig, snapshotConfig, eventConfig, cleanupConfig,
+                shutdownTimeout);
     }
 
     @Override
@@ -114,6 +125,7 @@ public String toString() {
                 "supervisorConfig=" + supervisorConfig +
                 ", activityCheckConfig=" + activityCheckConfig +
                 ", snapshotConfig=" + snapshotConfig +
+                ", eventConfig=" + eventConfig +
                 ", cleanupConfig=" + cleanupConfig +
                 ", shutdownTimeout=" + shutdownTimeout +
                 "]";
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/ThingConfig.java b/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/ThingConfig.java
index 544dd82a3cd..d9dc9936518 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/ThingConfig.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/ThingConfig.java
@@ -18,6 +18,7 @@
 
 import org.eclipse.ditto.base.service.config.supervision.WithSupervisorConfig;
 import org.eclipse.ditto.internal.utils.config.KnownConfigValue;
+import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithActivityCheckConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithSnapshotConfig;
 import org.eclipse.ditto.internal.utils.persistentactors.cleanup.WithCleanupConfig;
@@ -29,6 +30,13 @@
 public interface ThingConfig extends WithSupervisorConfig, WithActivityCheckConfig, WithSnapshotConfig,
         WithCleanupConfig {
 
+    /**
+     * Returns the config of the thing event journal behaviour.
+     *
+     * @return the config.
+     */
+    EventConfig getEventConfig();
+
     /**
      * Get the timeout waiting for responses and acknowledgements during coordinated shutdown.
      *
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/StreamRequestingCommandEnforcement.java b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/StreamRequestingCommandEnforcement.java
new file mode 100644
index 00000000000..d837476ffee
--- /dev/null
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/StreamRequestingCommandEnforcement.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.ditto.things.service.enforcement;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+
+import org.eclipse.ditto.base.model.signals.Signal;
+import org.eclipse.ditto.base.model.signals.commands.CommandResponse;
+import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand;
+import org.eclipse.ditto.policies.api.Permission;
+import org.eclipse.ditto.policies.enforcement.AbstractEnforcementReloaded;
+import org.eclipse.ditto.policies.enforcement.EnforcementReloaded;
+import org.eclipse.ditto.policies.enforcement.PolicyEnforcer;
+import org.eclipse.ditto.policies.model.Permissions;
+import org.eclipse.ditto.policies.model.ResourceKey;
+import org.eclipse.ditto.things.model.ThingId;
+import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingNotAccessibleException;
+
+/**
+ * Authorizes {@link StreamingSubscriptionCommand}s and filters {@link CommandResponse}s.
+ */
+final class StreamRequestingCommandEnforcement
+        extends AbstractEnforcementReloaded, CommandResponse>
+        implements ThingEnforcementStrategy {
+
+    @Override
+    public boolean isApplicable(final Signal signal) {
+        return signal instanceof StreamingSubscriptionCommand;
+    }
+
+    @Override
+    public boolean responseIsApplicable(final CommandResponse commandResponse) {
+        return false;
+    }
+
+    @Override
+    public , R extends CommandResponse> EnforcementReloaded getEnforcement() {
+        return (EnforcementReloaded) this;
+    }
+
+    @Override
+    public CompletionStage> authorizeSignal(final StreamingSubscriptionCommand signal,
+            final PolicyEnforcer policyEnforcer) {
+
+        final ResourceKey resourceKey = ResourceKey.newInstance(signal.getResourceType(), signal.getResourcePath());
+        if (policyEnforcer.getEnforcer().hasUnrestrictedPermissions(resourceKey,
+                signal.getDittoHeaders().getAuthorizationContext(), Permissions.newInstance(Permission.READ))) {
+            return CompletableFuture.completedStage(
+                    ThingCommandEnforcement.addEffectedReadSubjectsToThingSignal(signal, policyEnforcer.getEnforcer())
+            );
+        } else {
+            return CompletableFuture.failedStage(
+                    ThingNotAccessibleException.newBuilder(ThingId.of(signal.getEntityId()))
+                            .dittoHeaders(signal.getDittoHeaders())
+                            .build());
+        }
+    }
+
+    @Override
+    public CompletionStage> authorizeSignalWithMissingEnforcer(
+            final StreamingSubscriptionCommand signal) {
+
+        return CompletableFuture.failedStage(ThingNotAccessibleException.newBuilder(ThingId.of(signal.getEntityId()))
+                .dittoHeaders(signal.getDittoHeaders())
+                .build());
+    }
+
+    @Override
+    public boolean shouldFilterCommandResponse(final CommandResponse commandResponse) {
+        return false;
+    }
+
+    @Override
+    public CompletionStage> filterResponse(final CommandResponse commandResponse,
+            final PolicyEnforcer policyEnforcer) {
+        return CompletableFuture.completedStage(commandResponse);
+    }
+}
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingCommandEnforcement.java b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingCommandEnforcement.java
index 85b7dbab646..66be66eed4b 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingCommandEnforcement.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingCommandEnforcement.java
@@ -37,6 +37,7 @@
 import org.eclipse.ditto.internal.utils.cacheloaders.config.AskWithRetryConfig;
 import org.eclipse.ditto.json.JsonFactory;
 import org.eclipse.ditto.json.JsonFieldSelector;
+import org.eclipse.ditto.json.JsonKey;
 import org.eclipse.ditto.json.JsonObject;
 import org.eclipse.ditto.json.JsonPointer;
 import org.eclipse.ditto.json.JsonPointerInvalidException;
@@ -492,7 +493,8 @@ private static boolean enforceMergeThingCommand(final Enforcer enforcer,
         } else if (enforcer.hasPartialPermissions(thingResourceKey, authorizationContext, Permission.WRITE)) {
             // in case of partial permissions at thingResourceKey level check all leaves of merge patch for
             // unrestricted permissions
-            final Set resourceKeys = calculateLeaves(command.getPath(), command.getValue());
+            final Set resourceKeys = calculateLeaves(command.getPath(),
+                    command.getEntity().orElseGet(command::getValue));
             return enforcer.hasUnrestrictedPermissions(resourceKeys, authorizationContext, Permission.WRITE);
         } else {
             // not even partial permission
@@ -503,7 +505,15 @@ private static boolean enforceMergeThingCommand(final Enforcer enforcer,
     private static Set calculateLeaves(final JsonPointer path, final JsonValue value) {
         if (value.isObject()) {
             return value.asObject().stream()
-                    .map(f -> calculateLeaves(path.append(f.getKey().asPointer()), f.getValue()))
+                    .map(f -> {
+                        final JsonKey key = f.getKey();
+                        if (isMergeWithNulledKeysByRegex(key, f.getValue())) {
+                            // if regex is contained, writing all below that path must be granted in the policy:
+                            return Set.of(PoliciesResourceType.thingResource(path));
+                        } else {
+                            return calculateLeaves(path.append(key.asPointer()), f.getValue());
+                        }
+                    })
                     .reduce(new HashSet<>(), ThingCommandEnforcement::addAll, ThingCommandEnforcement::addAll);
         } else {
             return Set.of(PoliciesResourceType.thingResource(path));
@@ -515,4 +525,15 @@ private static Set addAll(final Set result, final Set<
         return result;
     }
 
+    private static boolean isMergeWithNulledKeysByRegex(final JsonKey key, final JsonValue value) {
+        final String keyString = key.toString();
+        if (keyString.startsWith("{{") && keyString.endsWith("}}")) {
+            final String keyRegexWithoutCurly = keyString.substring(2, keyString.length() - 2).trim();
+            return keyRegexWithoutCurly.startsWith("/") && keyRegexWithoutCurly.endsWith("/") &&
+                    value.isNull();
+        } else {
+            return false;
+        }
+    }
+
 }
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcement.java b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcement.java
index a2462c528fe..ff969d271e7 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcement.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcement.java
@@ -40,7 +40,8 @@ public ThingEnforcement(final ActorRef policiesShardRegion, final ActorSystem ac
 
         enforcementStrategies = List.of(
                 new LiveSignalEnforcement(),
-                new ThingCommandEnforcement(actorSystem, policiesShardRegion, enforcementConfig)
+                new ThingCommandEnforcement(actorSystem, policiesShardRegion, enforcementConfig),
+                new StreamRequestingCommandEnforcement()
         );
     }
 
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcerActor.java b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcerActor.java
index 713bc5b9a99..f6d3bc955e8 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcerActor.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcerActor.java
@@ -37,8 +37,10 @@
 import org.eclipse.ditto.json.JsonObjectBuilder;
 import org.eclipse.ditto.json.JsonRuntimeException;
 import org.eclipse.ditto.policies.api.PoliciesValidator;
+import org.eclipse.ditto.policies.api.PolicyTag;
 import org.eclipse.ditto.policies.enforcement.AbstractEnforcementReloaded;
 import org.eclipse.ditto.policies.enforcement.AbstractPolicyLoadingEnforcerActor;
+import org.eclipse.ditto.policies.enforcement.Invalidatable;
 import org.eclipse.ditto.policies.enforcement.PolicyEnforcer;
 import org.eclipse.ditto.policies.enforcement.PolicyEnforcerProvider;
 import org.eclipse.ditto.policies.model.PoliciesModelFactory;
@@ -221,7 +223,24 @@ private CompletionStage> loadPolicyEnforcerForCreateThi
             final Policy defaultPolicy = getDefaultPolicy(createThing.getDittoHeaders(), createThing.getEntityId());
             policyCs = createPolicy(defaultPolicy, createThing);
         }
-        return policyCs.thenCompose(policy -> providePolicyEnforcer(policy.getEntityId().orElse(null)));
+        final String correlationId =
+                createThing.getDittoHeaders().getCorrelationId().orElse("unexpected:" + UUID.randomUUID());
+        return policyCs
+                .thenCompose(policy -> {
+                    if (policyEnforcerProvider instanceof Invalidatable invalidatable &&
+                            policy.getEntityId().isPresent() && policy.getRevision().isPresent()) {
+                        return invalidatable.invalidate(PolicyTag.of(policy.getEntityId().get(),
+                                        policy.getRevision().get().toLong()), correlationId, askWithRetryConfig.getAskTimeout())
+                                .thenApply(bool -> {
+                                    log.withCorrelationId(correlationId)
+                                            .debug("PolicyEnforcerCache invalidated. Previous entity was present: {}",
+                                                    bool);
+                                    return policy;
+                                });
+                    }
+                    return CompletableFuture.completedFuture(policy);
+                })
+                .thenCompose(policy -> providePolicyEnforcer(policy.getEntityId().orElse(null)));
     }
 
     private CompletionStage getCopiedPolicy(final String policyIdOrPlaceholder,
@@ -359,8 +378,10 @@ private CompletionStage createPolicy(final Policy policy, final CreateTh
 
     private Policy handleCreatePolicyResponse(final CreatePolicy createPolicy, final Object policyResponse,
             final CreateThing createThing) {
-
         if (policyResponse instanceof CreatePolicyResponse createPolicyResponse) {
+            createPolicyResponse.getPolicyCreated()
+                    .ifPresent(policy -> getContext().getParent().tell(new ThingPolicyCreated(createThing.getEntityId(),
+                            createPolicyResponse.getEntityId(), createPolicy.getDittoHeaders()), getSelf()));
             return createPolicyResponse.getPolicyCreated().orElseThrow();
         } else {
             if (shouldReportInitialPolicyCreationFailure(policyResponse)) {
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingPolicyCreated.java b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingPolicyCreated.java
new file mode 100644
index 00000000000..b4aad9af16c
--- /dev/null
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingPolicyCreated.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+
+package org.eclipse.ditto.things.service.enforcement;
+
+
+import org.eclipse.ditto.base.model.headers.DittoHeaders;
+import org.eclipse.ditto.policies.model.PolicyId;
+import org.eclipse.ditto.things.model.ThingId;
+
+/**
+ * Used by the {@link org.eclipse.ditto.things.service.enforcement.ThingEnforcerActor} to notify the
+ * ThingSupervisorActor that a policy was created in result of ThingCreate enforcement.
+ * @param thingId thingId of the thing for which policy is created
+ * @param policyId the policyId of the created policy
+ * @param dittoHeaders dittoHeaders containing the correlationId of the initial command
+ */
+ public record ThingPolicyCreated(ThingId thingId, PolicyId policyId, DittoHeaders dittoHeaders) {}
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/pre/ModifyToCreateThingTransformer.java b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/pre/ModifyToCreateThingTransformer.java
index 551fa3321c6..c4e42a88f0e 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/pre/ModifyToCreateThingTransformer.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/pre/ModifyToCreateThingTransformer.java
@@ -12,21 +12,28 @@
  */
 package org.eclipse.ditto.things.service.enforcement.pre;
 
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 
+import javax.annotation.Nullable;
+
 import org.eclipse.ditto.base.model.signals.Signal;
 import org.eclipse.ditto.base.service.signaltransformer.SignalTransformer;
 import org.eclipse.ditto.json.JsonObject;
+import org.eclipse.ditto.things.model.Thing;
+import org.eclipse.ditto.things.model.ThingsModelFactory;
 import org.eclipse.ditto.things.model.signals.commands.modify.CreateThing;
+import org.eclipse.ditto.things.model.signals.commands.modify.MergeThing;
 import org.eclipse.ditto.things.model.signals.commands.modify.ModifyThing;
+import org.eclipse.ditto.things.model.signals.commands.modify.ThingModifyCommand;
 
 import com.typesafe.config.Config;
 
 import akka.actor.ActorSystem;
 
 /**
- * Transforms a ModifyThing into a CreateThing if the thing does not exist already.
+ * Transforms a ModifyThing and a MergeThing command into a CreateThing if the thing does not exist already.
  */
 public final class ModifyToCreateThingTransformer implements SignalTransformer {
 
@@ -42,28 +49,42 @@ public final class ModifyToCreateThingTransformer implements SignalTransformer {
 
     @Override
     public CompletionStage> apply(final Signal signal) {
+        return calculateInputParams(signal)
+                .map(input -> existenceChecker.checkExistence(input.thingModifyCommand())
+                        .thenApply(exists -> {
+                            if (Boolean.FALSE.equals(exists)) {
+                                final var newThing = input.thing().toBuilder()
+                                        .setId(input.thingModifyCommand().getEntityId())
+                                        .build();
+                                return CreateThing.of(newThing, input.initialPolicy(), input.policyIdOrPlaceholder(),
+                                        input.thingModifyCommand().getDittoHeaders());
+                            } else {
+                                return (Signal) input.thingModifyCommand();
+                            }
+                        })
+                ).orElse(CompletableFuture.completedStage(signal));
+    }
+
+    private static Optional calculateInputParams(final Signal signal) {
         if (signal instanceof ModifyThing modifyThing) {
-            return existenceChecker.checkExistence(modifyThing)
-                    .thenApply(exists -> {
-                        if (Boolean.FALSE.equals(exists)) {
-                            final JsonObject initialPolicy = modifyThing.getInitialPolicy().orElse(null);
-                            final String policyIdOrPlaceholder = modifyThing.getPolicyIdOrPlaceholder().orElse(null);
-                            final var newThing = modifyThing.getThing().toBuilder()
-                                    .setId(modifyThing.getEntityId())
-                                    .build();
-                            return CreateThing.of(
-                                    newThing,
-                                    initialPolicy,
-                                    policyIdOrPlaceholder,
-                                    modifyThing.getDittoHeaders()
-                            );
-                        } else {
-                            return modifyThing;
-                        }
-                    });
+            return Optional.of(new InputParams(modifyThing,
+                    modifyThing.getThing(),
+                    modifyThing.getInitialPolicy().orElse(null),
+                    modifyThing.getPolicyIdOrPlaceholder().orElse(null)
+            ));
+        } else if (signal instanceof MergeThing mergeThing && mergeThing.getPath().isEmpty()) {
+            final JsonObject mergeThingObject = mergeThing.getEntity().orElseGet(mergeThing::getValue).asObject();
+            return Optional.of(new InputParams(mergeThing,
+                    ThingsModelFactory.newThing(mergeThingObject),
+                    mergeThing.getInitialPolicy().orElse(null),
+                    mergeThing.getPolicyIdOrPlaceholder().orElse(null)
+            ));
         } else {
-            return CompletableFuture.completedStage(signal);
+            return Optional.empty();
         }
     }
 
+    private record InputParams(ThingModifyCommand thingModifyCommand, Thing thing,
+            @Nullable JsonObject initialPolicy,
+            @Nullable String policyIdOrPlaceholder) {}
 }
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/pre/ThingExistenceChecker.java b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/pre/ThingExistenceChecker.java
index 05db6712332..e368e048d06 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/pre/ThingExistenceChecker.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/pre/ThingExistenceChecker.java
@@ -23,7 +23,7 @@
 import org.eclipse.ditto.policies.model.PolicyId;
 import org.eclipse.ditto.things.api.ThingsMessagingConstants;
 import org.eclipse.ditto.things.model.ThingId;
-import org.eclipse.ditto.things.model.signals.commands.modify.ModifyThing;
+import org.eclipse.ditto.things.model.signals.commands.modify.ThingModifyCommand;
 
 import com.github.benmanes.caffeine.cache.AsyncCacheLoader;
 
@@ -62,7 +62,7 @@ private AsyncCacheLoader> getThingIdLoader(
                 thingsShardRegion);
     }
 
-    public CompletionStage checkExistence(final ModifyThing signal) {
+    public CompletionStage checkExistence(final ThingModifyCommand signal) {
         try {
             return thingIdLoader.asyncLoad(signal.getEntityId(),
                             actorSystem.dispatchers().lookup(ENFORCEMENT_CACHE_DISPATCHER))
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActor.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActor.java
index 9e682bc0b88..41bb3dcadde 100755
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActor.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActor.java
@@ -12,6 +12,9 @@
  */
 package org.eclipse.ditto.things.service.persistence.actors;
 
+import java.time.Instant;
+import java.util.concurrent.CompletionStage;
+
 import javax.annotation.Nullable;
 
 import org.eclipse.ditto.base.model.acks.DittoAcknowledgementLabel;
@@ -24,6 +27,7 @@
 import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig;
+import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal;
 import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceActor;
 import org.eclipse.ditto.internal.utils.persistentactors.commands.CommandStrategy;
 import org.eclipse.ditto.internal.utils.persistentactors.commands.DefaultContext;
@@ -38,6 +42,7 @@
 import org.eclipse.ditto.things.model.ThingId;
 import org.eclipse.ditto.things.model.ThingLifecycle;
 import org.eclipse.ditto.things.model.ThingsModelFactory;
+import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingHistoryNotAccessibleException;
 import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingNotAccessibleException;
 import org.eclipse.ditto.things.model.signals.commands.modify.CreateThing;
 import org.eclipse.ditto.things.model.signals.commands.query.RetrieveThing;
@@ -83,10 +88,11 @@ public final class ThingPersistenceActor
 
     @SuppressWarnings("unused")
     private ThingPersistenceActor(final ThingId thingId,
+            final MongoReadJournal mongoReadJournal,
             final DistributedPub> distributedPub,
             @Nullable final ActorRef searchShardRegionProxy) {
 
-        super(thingId);
+        super(thingId, mongoReadJournal);
         final DittoThingsConfig thingsConfig = DittoThingsConfig.of(
                 DefaultScopedConfig.dittoScoped(getContext().getSystem().settings().config())
         );
@@ -99,29 +105,47 @@ private ThingPersistenceActor(final ThingId thingId,
      * Creates Akka configuration object {@link Props} for this ThingPersistenceActor.
      *
      * @param thingId the Thing ID this Actor manages.
+     * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the thing.
      * @param distributedPub the distributed-pub access to publish thing events.
      * @return the Akka configuration Props object
      */
     public static Props props(final ThingId thingId,
+            final MongoReadJournal mongoReadJournal,
             final DistributedPub> distributedPub,
             @Nullable final ActorRef searchShardRegionProxy) {
 
-        return Props.create(ThingPersistenceActor.class, thingId, distributedPub, searchShardRegionProxy);
+        return Props.create(ThingPersistenceActor.class, thingId, mongoReadJournal, distributedPub,
+                searchShardRegionProxy);
     }
 
     @Override
     public void onQuery(final Command command, final WithDittoHeaders response) {
+        final ActorRef sender = getSender();
+        doOnQuery(command, response, sender);
+    }
+
+    @Override
+    public void onStagedQuery(final Command command, final CompletionStage response) {
+        final ActorRef sender = getSender();
+        response.thenAccept(r -> doOnQuery(command, r, sender));
+    }
+
+    private void doOnQuery(final Command command, final WithDittoHeaders response, final ActorRef sender) {
         if (response.getDittoHeaders().didLiveChannelConditionMatch()) {
             final var liveChannelTimeoutStrategy = response.getDittoHeaders()
                     .getLiveChannelTimeoutStrategy()
                     .orElse(LiveChannelTimeoutStrategy.FAIL);
             if (liveChannelTimeoutStrategy != LiveChannelTimeoutStrategy.USE_TWIN &&
-                    response instanceof ThingQueryCommandResponse queryResponse) {
-                super.onQuery(command, queryResponse.setEntity(JsonFactory.nullLiteral()));
+                    response instanceof ThingQueryCommandResponse queryResponse &&
+                    command.getDittoHeaders().isResponseRequired()) {
+                notifySender(sender, queryResponse.setEntity(JsonFactory.nullLiteral()));
                 return;
             }
         }
-        super.onQuery(command, response);
+
+        if (command.getDittoHeaders().isResponseRequired()) {
+            notifySender(sender, response);
+        }
     }
 
     @Override
@@ -193,6 +217,16 @@ protected DittoRuntimeExceptionBuilder newNotAccessibleExceptionBuilder() {
         return ThingNotAccessibleException.newBuilder(entityId);
     }
 
+    @Override
+    protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final long revision) {
+        return ThingHistoryNotAccessibleException.newBuilder(entityId, revision);
+    }
+
+    @Override
+    protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final Instant timestamp) {
+        return ThingHistoryNotAccessibleException.newBuilder(entityId, timestamp);
+    }
+
     @Override
     protected void recoveryCompleted(final RecoveryCompleted event) {
         if (entity != null) {
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorPropsFactory.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorPropsFactory.java
index 7845e202f82..a5d5777896d 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorPropsFactory.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorPropsFactory.java
@@ -14,8 +14,9 @@
 
 import javax.annotation.Nullable;
 
-import org.eclipse.ditto.things.model.ThingId;
+import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal;
 import org.eclipse.ditto.internal.utils.pubsub.DistributedPub;
+import org.eclipse.ditto.things.model.ThingId;
 import org.eclipse.ditto.things.model.signals.events.ThingEvent;
 
 import akka.actor.ActorRef;
@@ -31,10 +32,11 @@ public interface ThingPersistenceActorPropsFactory {
      * Create Props of thing-persistence-actor from thing ID and distributed-pub access for event publishing.
      *
      * @param thingId the thing ID.
+     * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the thing.
      * @param distributedPub the distributed-pub access.
      * @param searchShardRegionProxy the proxy of the shard region of search updaters.
      * @return Props of the thing-persistence-actor.
      */
-    Props props(ThingId thingId, DistributedPub> distributedPub,
-            @Nullable final ActorRef searchShardRegionProxy);
+    Props props(ThingId thingId, MongoReadJournal mongoReadJournal, DistributedPub> distributedPub,
+            @Nullable ActorRef searchShardRegionProxy);
 }
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingSupervisorActor.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingSupervisorActor.java
index d7c732b1c71..a9f3ec25dcd 100755
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingSupervisorActor.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingSupervisorActor.java
@@ -12,17 +12,24 @@
  */
 package org.eclipse.ditto.things.service.persistence.actors;
 
+
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.Objects;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.CompletionStage;
 
 import javax.annotation.Nullable;
 
 import org.eclipse.ditto.base.model.acks.DittoAcknowledgementLabel;
 import org.eclipse.ditto.base.model.auth.AuthorizationContext;
+import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException;
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder;
+import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition;
+import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.base.model.headers.WithDittoHeaders;
 import org.eclipse.ditto.base.model.signals.Signal;
 import org.eclipse.ditto.base.model.signals.commands.Command;
@@ -30,10 +37,13 @@
 import org.eclipse.ditto.base.model.signals.events.Event;
 import org.eclipse.ditto.base.service.actors.ShutdownBehaviour;
 import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOffConfig;
+import org.eclipse.ditto.base.service.config.supervision.LocalAskTimeoutConfig;
+import org.eclipse.ditto.internal.utils.cacheloaders.AskWithRetry;
 import org.eclipse.ditto.internal.utils.cluster.ShardRegionProxyActorFactory;
 import org.eclipse.ditto.internal.utils.cluster.StopShardedActor;
 import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig;
 import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces;
+import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal;
 import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor;
 import org.eclipse.ditto.internal.utils.persistentactors.TargetActorWithMessage;
 import org.eclipse.ditto.internal.utils.pubsub.DistributedPub;
@@ -41,10 +51,12 @@
 import org.eclipse.ditto.policies.api.PoliciesMessagingConstants;
 import org.eclipse.ditto.policies.enforcement.PolicyEnforcerProvider;
 import org.eclipse.ditto.policies.enforcement.config.DefaultEnforcementConfig;
+import org.eclipse.ditto.policies.model.signals.commands.modify.DeletePolicy;
 import org.eclipse.ditto.things.api.ThingsMessagingConstants;
 import org.eclipse.ditto.things.model.ThingId;
 import org.eclipse.ditto.things.model.signals.commands.ThingCommandResponse;
 import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingUnavailableException;
+import org.eclipse.ditto.things.model.signals.commands.modify.CreateThing;
 import org.eclipse.ditto.things.model.signals.commands.query.RetrieveThing;
 import org.eclipse.ditto.things.model.signals.commands.query.RetrieveThingResponse;
 import org.eclipse.ditto.things.model.signals.commands.query.ThingQueryCommand;
@@ -52,6 +64,7 @@
 import org.eclipse.ditto.things.service.common.config.DittoThingsConfig;
 import org.eclipse.ditto.things.service.enforcement.ThingEnforcement;
 import org.eclipse.ditto.things.service.enforcement.ThingEnforcerActor;
+import org.eclipse.ditto.things.service.enforcement.ThingPolicyCreated;
 import org.eclipse.ditto.thingsearch.api.ThingsSearchConstants;
 
 import akka.actor.ActorKilledException;
@@ -61,6 +74,7 @@
 import akka.actor.Props;
 import akka.japi.pf.FI;
 import akka.japi.pf.ReceiveBuilder;
+import akka.pattern.AskTimeoutException;
 import akka.stream.Materializer;
 import akka.stream.javadsl.Keep;
 import akka.stream.javadsl.Sink;
@@ -93,6 +107,8 @@ public final class ThingSupervisorActor extends AbstractPersistenceSupervisor modifyTargetActorCommandResponse(final Signal<
                             pair.response() instanceof RetrieveThingResponse retrieveThingResponse) {
                         return inlinePolicyEnrichment.enrichPolicy(retrieveThing, retrieveThingResponse)
                                 .map(Object.class::cast);
+                    } else if (RollbackCreatedPolicy.shouldRollbackBasedOnTargetActorResponse(pair.command(), pair.response())) {
+                        final CompletableFuture responseF = new CompletableFuture<>();
+                        getSelf().tell(RollbackCreatedPolicy.of(pair.command(), pair.response(), responseF), getSelf());
+                        return Source.completionStage(responseF);
                     } else {
                         return Source.single(pair.response());
                     }
@@ -309,6 +334,57 @@ protected CompletionStage modifyTargetActorCommandResponse(final Signal<
                 .run(materializer);
     }
 
+    @Override
+    protected CompletableFuture handleTargetActorAndEnforcerException(final Signal signal, final Throwable throwable) {
+        if (RollbackCreatedPolicy.shouldRollbackBasedOnException(signal, throwable)) {
+            log.withCorrelationId(signal)
+                    .info("Target actor exception received: <{}>. " +
+                            "Sending RollbackCreatedPolicy msg to self, potentially rolling back a created policy.",
+                            throwable.getClass().getSimpleName());
+            final CompletableFuture responseFuture = new CompletableFuture<>();
+            getSelf().tell(RollbackCreatedPolicy.of(signal, throwable, responseFuture), getSelf());
+            return responseFuture;
+        } else {
+            log.withCorrelationId(signal)
+                    .debug("Target actor exception received: <{}>", throwable.getClass().getSimpleName());
+            return CompletableFuture.failedFuture(throwable);
+        }
+    }
+
+    private void handleRollbackCreatedPolicy(final RollbackCreatedPolicy rollback) {
+        final String correlationId = rollback.initialCommand().getDittoHeaders().getCorrelationId()
+                .orElse("unexpected:" + UUID.randomUUID());
+        if (policyCreatedEvent != null) {
+            log.withCorrelationId(correlationId)
+                    .warning("Rolling back created policy as consequence of received RollbackCreatedPolicy " +
+                            "message: {}", rollback);
+            final DittoHeaders dittoHeaders = DittoHeaders.newBuilder()
+                    .correlationId(correlationId)
+                    .putHeader(DittoHeaderDefinition.DITTO_SUDO.getKey(), "true")
+                    .build();
+            final DeletePolicy deletePolicy = DeletePolicy.of(policyCreatedEvent.policyId(), dittoHeaders);
+            AskWithRetry.askWithRetry(policiesShardRegion, deletePolicy,
+                    enforcementConfig.getAskWithRetryConfig(),
+                    getContext().system(), response -> {
+                        log.withCorrelationId(correlationId)
+                                .info("Policy <{}> deleted after rolling back it's creation. " +
+                                        "Policies shard region response: <{}>", deletePolicy.getEntityId(), response);
+                        rollback.completeInitialResponse();
+                        return response;
+                    }).exceptionally(throwable -> {
+                log.withCorrelationId(correlationId).error(throwable, "Failed to rollback Policy Create");
+                rollback.completeInitialResponse();
+                return null;
+            });
+
+        } else {
+            log.withCorrelationId(correlationId)
+                    .debug("Not initiating policy rollback as none was created.");
+            rollback.completeInitialResponse();
+        }
+        policyCreatedEvent = null;
+    }
+
     @Override
     protected ThingId getEntityId() throws Exception {
         return ThingId.of(URLDecoder.decode(getSelf().path().name(), StandardCharsets.UTF_8));
@@ -317,7 +393,7 @@ protected ThingId getEntityId() throws Exception {
     @Override
     protected Props getPersistenceActorProps(final ThingId entityId) {
         assert thingPersistenceActorPropsFactory != null;
-        return thingPersistenceActorPropsFactory.props(entityId, distributedPubThingEventsForTwin,
+        return thingPersistenceActorPropsFactory.props(entityId, mongoReadJournal, distributedPubThingEventsForTwin,
                 searchShardRegionProxy);
     }
 
@@ -358,6 +434,14 @@ protected ExponentialBackOffConfig getExponentialBackOffConfig() {
                 .getExponentialBackOffConfig();
     }
 
+    @Override
+    protected LocalAskTimeoutConfig getLocalAskTimeoutConfig() {
+        return DittoThingsConfig.of(DefaultScopedConfig.dittoScoped(getContext().getSystem().settings().config()))
+                .getThingConfig()
+                .getSupervisorConfig()
+                .getLocalAskTimeoutConfig();
+    }
+
     @Override
     protected void stopShardedActor(final StopShardedActor trigger) {
         super.stopShardedActor(trigger);
@@ -371,6 +455,10 @@ protected Receive activeBehaviour(final Runnable matchProcessNextTwinMessageBeha
             final FI.UnitApply matchAnyBehavior) {
         return ReceiveBuilder.create()
                 .matchEquals(Control.SHUTDOWN_TIMEOUT, this::shutdownActor)
+                .match(ThingPolicyCreated.class, msg -> {
+                    log.withCorrelationId(msg.dittoHeaders()).info("ThingPolicyCreated msg received: <{}>", msg.policyId());
+                    this.policyCreatedEvent = msg;
+                }).match(RollbackCreatedPolicy.class, this::handleRollbackCreatedPolicy)
                 .build()
                 .orElse(super.activeBehaviour(matchProcessNextTwinMessageBehavior, matchAnyBehavior));
     }
@@ -387,4 +475,79 @@ private enum Control {
         SHUTDOWN_TIMEOUT
     }
 
+    /**
+     * Used from the {@link org.eclipse.ditto.things.service.persistence.actors.ThingSupervisorActor} to signal itself
+     * to delete an already created policy because of a failure in creating a thing
+     * @param initialCommand the initial command that triggered the creation of a thing and policy
+     * @param response the response from the thing persistence actor
+     * @param responseFuture a future that when completed with the response from the thing persistence actor the response
+     * will be sent to the initial sender.
+     */
+    private record RollbackCreatedPolicy(Signal initialCommand, Object response, CompletableFuture responseFuture) {
+
+        /**
+         * Initialises an instance of {@link org.eclipse.ditto.things.service.persistence.actors.ThingSupervisorActor.RollbackCreatedPolicy}
+         * @param initialCommand the initial initialCommand that triggered the creation of a thing and policy
+         * @param response the response from the thing persistence actor
+         * @param responseFuture a future that when completed with the response from the thing persistence actor the response
+         * will be sent to the initial sender.
+         * @return an instance of {@link org.eclipse.ditto.things.service.persistence.actors.ThingSupervisorActor.RollbackCreatedPolicy}
+         */
+        public static RollbackCreatedPolicy of(final Signal initialCommand, final Object response,
+                final CompletableFuture responseFuture) {
+            return new RollbackCreatedPolicy(initialCommand, response, responseFuture);
+        }
+
+        /**
+         * Evaluates if a failure in the creation of a thing should lead to deleting of that thing's policy.
+         * Should be used only to evaluate exceptions from the target actor not the enforcement actor.
+         * @param command the initial command.
+         * @param response the response from the {@link org.eclipse.ditto.things.service.persistence.actors.ThingPersistenceActor}.
+         * @return if the thing's policy is to be deleted.
+         */
+        private static boolean shouldRollbackBasedOnTargetActorResponse(final Signal command, @Nullable final Object response) {
+            return command instanceof CreateThing && response instanceof DittoRuntimeException;
+        }
+
+        /**
+         * Evaluates if a failure in the creation of a thing should lead to deleting of that thing's policy.
+         * @param signal the initial signal.
+         * @param throwable the throwable received from the Persistence Actor
+         * @return if the thing's policy is to be deleted.
+         */
+        private static boolean shouldRollbackBasedOnException(final Signal signal, @Nullable final Throwable throwable) {
+                return signal instanceof CreateThing && ((throwable instanceof CompletionException ce1 &&  ce1.getCause() instanceof ThingUnavailableException)
+                        || throwable instanceof AskTimeoutException
+                        || (throwable instanceof CompletionException ce && ce.getCause() instanceof AskTimeoutException)
+                );
+        }
+
+        /**
+         * Completes the responseFuture with the response which in turn should send the Persistence actor response to
+         * the initial sender. If an additional exception occurs during policy rollback the responseFuture will be
+         * completed with that exception and adding the target actor exception as suppressed warning.
+         *
+         * @param throwable the additional optional exception occurred during the rollback process.
+         */
+        private void completeInitialResponse(@Nullable final Throwable throwable) {
+            if (response instanceof Throwable t) {
+                if (throwable != null) {
+                    throwable.addSuppressed(t);
+                    responseFuture.completeExceptionally(throwable);
+                } else {
+                    responseFuture.completeExceptionally(t);
+                }
+            } else {
+                responseFuture.complete(response);
+            }
+        }
+
+        /**
+         * Completes the responseFuture with the response which in turn should send the Persistence actor response to
+         * the initial sender.
+         */
+        private void completeInitialResponse() {
+            completeInitialResponse(null);
+        }
+    }
 }
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingsPersistenceStreamingActorCreator.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingsPersistenceStreamingActorCreator.java
index bef308fc3e1..bae2ae9d751 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingsPersistenceStreamingActorCreator.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingsPersistenceStreamingActorCreator.java
@@ -31,9 +31,9 @@
 public final class ThingsPersistenceStreamingActorCreator {
 
     /**
-     * The name of the snapshot streaming actor.
+     * The name of the streaming actor.
      */
-    public static final String SNAPSHOT_STREAMING_ACTOR_NAME = THINGS_PERSISTENCE_STREAMING_ACTOR_NAME;
+    public static final String STREAMING_ACTOR_NAME = THINGS_PERSISTENCE_STREAMING_ACTOR_NAME;
 
     private static final Pattern PERSISTENCE_ID_PATTERN = Pattern.compile(ThingPersistenceActor.PERSISTENCE_ID_PREFIX);
 
@@ -42,16 +42,16 @@ private ThingsPersistenceStreamingActorCreator() {
     }
 
     /**
-     * Create an actor that streams from the snapshot store.
+     * Create an actor that streams from the snapshot store and the event journal.
      *
      * @param actorCreator function to create a named actor with.
      * @return a reference of the created actor.
      */
-    public static ActorRef startSnapshotStreamingActor(final BiFunction actorCreator) {
+    public static ActorRef startPersistenceStreamingActor(final BiFunction actorCreator) {
         final var props = SnapshotStreamingActor.props(ThingsPersistenceStreamingActorCreator::pid2EntityId,
                 ThingsPersistenceStreamingActorCreator::entityId2Pid);
 
-        return actorCreator.apply(SNAPSHOT_STREAMING_ACTOR_NAME, props);
+        return actorCreator.apply(STREAMING_ACTOR_NAME, props);
     }
 
     private static ThingId pid2EntityId(final String pid) {
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/CreateThingStrategy.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/CreateThingStrategy.java
index 211eacdebba..6d5d16102b7 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/CreateThingStrategy.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/CreateThingStrategy.java
@@ -18,6 +18,7 @@
 import java.time.Instant;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.concurrent.CompletionStage;
 
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
@@ -106,34 +107,40 @@ protected Result> doApply(final Context context,
             newThing = newThing.setPolicyId(PolicyId.of(context.getState()));
         }
 
+        final Instant now = Instant.now();
+
         final Thing finalNewThing = newThing;
-        newThing = wotThingDescriptionProvider.provideThingSkeletonForCreation(
+        final CompletionStage thingStage = wotThingDescriptionProvider.provideThingSkeletonForCreation(
                         command.getEntityId(),
                         newThing.getDefinition().orElse(null),
                         commandHeaders
                 )
-                .map(wotBasedThingSkeleton ->
-                        JsonFactory.mergeJsonValues(finalNewThing.toJson(), wotBasedThingSkeleton.toJson())
-                )
-                .filter(JsonValue::isObject)
-                .map(JsonValue::asObject)
-                .map(ThingsModelFactory::newThing)
-                .orElse(finalNewThing);
-
-        final Instant now = Instant.now();
-        final Thing newThingWithImplicits = newThing.toBuilder()
-                .setModified(now)
-                .setCreated(now)
-                .setRevision(nextRevision)
-                .setMetadata(metadata)
-                .build();
-        final ThingCreated thingCreated = ThingCreated.of(newThingWithImplicits, nextRevision, now, commandHeaders,
-                metadata);
-        final WithDittoHeaders response = appendETagHeaderIfProvided(command,
-                CreateThingResponse.of(newThingWithImplicits, commandHeaders),
-                newThingWithImplicits);
-
-        return newMutationResult(command, thingCreated, response, true, false);
+                .thenApply(opt -> opt.map(wotBasedThingSkeleton ->
+                                        JsonFactory.mergeJsonValues(finalNewThing.toJson(), wotBasedThingSkeleton.toJson())
+                                )
+                                .filter(JsonValue::isObject)
+                                .map(JsonValue::asObject)
+                                .map(ThingsModelFactory::newThing)
+                                .orElse(finalNewThing)
+                ).thenApply(enhancedThing -> enhancedThing.toBuilder()
+                        .setModified(now)
+                        .setCreated(now)
+                        .setRevision(nextRevision)
+                        .setMetadata(metadata)
+                        .build()
+                );
+
+        final CompletionStage> eventStage =
+                thingStage.thenApply(newThingWithImplicits ->
+                        ThingCreated.of(newThingWithImplicits, nextRevision, now, commandHeaders, metadata)
+                );
+
+        final CompletionStage responseStage = thingStage.thenApply(newThingWithImplicits ->
+                appendETagHeaderIfProvided(command, CreateThingResponse.of(newThingWithImplicits, commandHeaders),
+                        newThingWithImplicits)
+        );
+
+        return newMutationResult(command, eventStage, responseStage, true, false);
     }
 
     private Thing handleCommandVersion(final Context context, final JsonSchemaVersion version,
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/MergeThingStrategy.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/MergeThingStrategy.java
index ae840a0da28..04470101746 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/MergeThingStrategy.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/MergeThingStrategy.java
@@ -93,7 +93,7 @@ private Result> applyMergeCommand(final Context context,
         // (this is required e.g. for updating the search-index)
         final DittoHeaders dittoHeaders = command.getDittoHeaders();
         final JsonPointer path = command.getPath();
-        final JsonValue value = command.getValue();
+        final JsonValue value = command.getEntity().orElseGet(command::getValue);
 
         final Thing mergedThing = wrapException(() -> mergeThing(context, command, thing, eventTs, nextRevision),
                 command.getDittoHeaders());
@@ -109,7 +109,8 @@ private Result> applyMergeCommand(final Context context,
     private Thing mergeThing(final Context context, final MergeThing command, final Thing thing,
             final Instant eventTs, final long nextRevision) {
         final JsonObject existingThingJson = thing.toJson(FieldType.all());
-        final JsonMergePatch jsonMergePatch = JsonMergePatch.of(command.getPath(), command.getValue());
+        final JsonMergePatch jsonMergePatch = JsonMergePatch.of(command.getPath(),
+                command.getEntity().orElseGet(command::getValue));
         final JsonObject mergedJson = jsonMergePatch.applyOn(existingThingJson).asObject();
 
         ThingCommandSizeValidator.getInstance().ensureValidSize(
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeatureStrategy.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeatureStrategy.java
index 91466b67fd5..019cdb59512 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeatureStrategy.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeatureStrategy.java
@@ -15,6 +15,7 @@
 import java.util.LinkedHashSet;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CompletionStage;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -121,49 +122,50 @@ private Result> getCreateResult(final Context context, fi
 
         final DittoHeaders dittoHeaders = command.getDittoHeaders();
 
-        Feature feature = command.getFeature();
-        final Feature finalNewFeature = feature;
-        feature = wotThingDescriptionProvider.provideFeatureSkeletonForCreation(
+        final Feature finalNewFeature = command.getFeature();
+        final CompletionStage featureStage = wotThingDescriptionProvider.provideFeatureSkeletonForCreation(
                         finalNewFeature.getId(),
                         finalNewFeature.getDefinition().orElse(null),
                         dittoHeaders
                 )
-                .map(wotBasedFeatureSkeleton -> {
-                        final Optional mergedDefinition =
-                                wotBasedFeatureSkeleton.getDefinition()
-                                        .map(def -> {
-                                                final Set identifiers = Stream.concat(
-                                                        wotBasedFeatureSkeleton.getDefinition()
-                                                                .map(FeatureDefinition::stream)
-                                                                .orElse(Stream.empty()),
-                                                        finalNewFeature.getDefinition()
-                                                                .map(FeatureDefinition::stream)
-                                                                .orElse(Stream.empty())
-                                                ).collect(Collectors.toCollection(LinkedHashSet::new));
-                                                return ThingsModelFactory.newFeatureDefinition(identifiers);
-                                        })
-                                        .or(finalNewFeature::getDefinition);
-
-                        return mergedDefinition.map(definitionIdentifiers -> JsonFactory.mergeJsonValues(
-                                finalNewFeature.setDefinition(definitionIdentifiers).toJson(),
-                                wotBasedFeatureSkeleton.toJson()
-                        )).orElseGet(() -> JsonFactory.mergeJsonValues(finalNewFeature.toJson(),
-                                wotBasedFeatureSkeleton.toJson())
-                        );
-                })
-                .filter(JsonValue::isObject)
-                .map(JsonValue::asObject)
-                .map(ThingsModelFactory::newFeatureBuilder)
-                .map(b -> b.useId(finalNewFeature.getId()).build())
-                .orElse(finalNewFeature);
-
-        final ThingEvent event =
-                FeatureCreated.of(command.getEntityId(), feature, nextRevision, getEventTimestamp(), dittoHeaders,
-                        metadata);
-        final WithDittoHeaders response = appendETagHeaderIfProvided(command,
-                ModifyFeatureResponse.created(context.getState(), feature, dittoHeaders), thing);
-
-        return ResultFactory.newMutationResult(command, event, response);
+                .thenApply(opt -> opt.map(wotBasedFeatureSkeleton -> wotBasedFeatureSkeleton.getDefinition()
+                                .map(def -> {
+                                    final Set identifiers = Stream.concat(
+                                            wotBasedFeatureSkeleton.getDefinition()
+                                                    .map(FeatureDefinition::stream)
+                                                    .orElse(Stream.empty()),
+                                            finalNewFeature.getDefinition()
+                                                    .map(FeatureDefinition::stream)
+                                                    .orElse(Stream.empty())
+                                    ).collect(Collectors.toCollection(LinkedHashSet::new));
+                                    return ThingsModelFactory.newFeatureDefinition(identifiers);
+                                })
+                                .or(finalNewFeature::getDefinition)
+                                .map(definitionIdentifiers -> JsonFactory.mergeJsonValues(
+                                        finalNewFeature.setDefinition(definitionIdentifiers).toJson(),
+                                        wotBasedFeatureSkeleton.toJson()
+                                ))
+                                .orElseGet(() -> JsonFactory.mergeJsonValues(finalNewFeature.toJson(),
+                                        wotBasedFeatureSkeleton.toJson())
+                                ))
+                        .filter(JsonValue::isObject)
+                        .map(JsonValue::asObject)
+                        .map(ThingsModelFactory::newFeatureBuilder)
+                        .map(b -> b.useId(finalNewFeature.getId()).build())
+                        .orElse(finalNewFeature)
+                );
+
+        final CompletionStage> eventStage =
+                featureStage.thenApply(feature -> FeatureCreated.of(command.getEntityId(), feature, nextRevision,
+                        getEventTimestamp(), dittoHeaders,
+                        metadata));
+
+        final CompletionStage response = featureStage.thenApply(modFeature ->
+                appendETagHeaderIfProvided(command,
+                        ModifyFeatureResponse.created(context.getState(), modFeature, dittoHeaders), thing)
+        );
+
+        return ResultFactory.newMutationResult(command, eventStage, response);
     }
 
     @Override
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeaturesStrategy.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeaturesStrategy.java
index 70a1b4506ff..f222a2a487e 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeaturesStrategy.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeaturesStrategy.java
@@ -13,8 +13,12 @@
 package org.eclipse.ditto.things.service.persistence.actors.strategies.commands;
 
 import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -31,6 +35,7 @@
 import org.eclipse.ditto.json.JsonObject;
 import org.eclipse.ditto.json.JsonValue;
 import org.eclipse.ditto.things.model.DefinitionIdentifier;
+import org.eclipse.ditto.things.model.Feature;
 import org.eclipse.ditto.things.model.FeatureDefinition;
 import org.eclipse.ditto.things.model.Features;
 import org.eclipse.ditto.things.model.Thing;
@@ -113,19 +118,17 @@ private Result> getCreateResult(final Context context, fi
 
         final DittoHeaders dittoHeaders = command.getDittoHeaders();
 
-        final Features features = ThingsModelFactory.newFeatures(command.getFeatures()
+        final List> featureStages = command.getFeatures()
                 .stream()
                 .map(feature -> wotThingDescriptionProvider.provideFeatureSkeletonForCreation(
-                                feature.getId(),
-                                feature.getDefinition().orElse(null),
-                                dittoHeaders
-                        )
-                        .map(wotBasedFeatureSkeleton -> {
-                                final Optional mergedDefinition =
-                                        wotBasedFeatureSkeleton.getDefinition()
+                                        feature.getId(),
+                                        feature.getDefinition().orElse(null),
+                                        dittoHeaders
+                                )
+                                .thenApply(opt -> opt.map(wotBasedFSkel -> wotBasedFSkel.getDefinition()
                                                 .map(def -> {
                                                     final Set identifiers = Stream.concat(
-                                                            wotBasedFeatureSkeleton.getDefinition()
+                                                            wotBasedFSkel.getDefinition()
                                                                     .map(FeatureDefinition::stream)
                                                                     .orElse(Stream.empty()),
                                                             feature.getDefinition()
@@ -134,32 +137,45 @@ private Result> getCreateResult(final Context context, fi
                                                     ).collect(Collectors.toCollection(LinkedHashSet::new));
                                                     return ThingsModelFactory.newFeatureDefinition(identifiers);
                                                 })
-                                                .or(feature::getDefinition);
-
-                                return mergedDefinition.map(definitionIdentifiers -> JsonFactory.mergeJsonValues(
-                                        feature.setDefinition(definitionIdentifiers).toJson(),
-                                        wotBasedFeatureSkeleton.toJson()
-                                )).orElseGet(() -> JsonFactory.mergeJsonValues(feature.toJson(),
-                                        wotBasedFeatureSkeleton.toJson())
-                                );
-
-                        })
-                        .filter(JsonValue::isObject)
-                        .map(JsonValue::asObject)
-                        .map(ThingsModelFactory::newFeatureBuilder)
-                        .map(b -> b.useId(feature.getId()).build())
-                        .orElse(feature)
+                                                .or(feature::getDefinition).map(definitionIdentifiers -> JsonFactory.mergeJsonValues(
+                                                        feature.setDefinition(definitionIdentifiers).toJson(),
+                                                        wotBasedFSkel.toJson()
+                                                )).orElseGet(() -> JsonFactory.mergeJsonValues(feature.toJson(),
+                                                        wotBasedFSkel.toJson())
+                                                ))
+                                        .filter(JsonValue::isObject)
+                                        .map(JsonValue::asObject)
+                                        .map(ThingsModelFactory::newFeatureBuilder)
+                                        .map(b -> b.useId(feature.getId()).build())
+                                        .orElse(feature)
+                                )
+                )
+                .toList();
+
+        final CompletableFuture featuresStage =
+                CompletableFuture.allOf(featureStages.toArray(new CompletableFuture[0]))
+                        .thenApply(aVoid ->
+                                ThingsModelFactory.newFeatures(
+                                        featureStages.stream()
+                                                .map(CompletionStage::toCompletableFuture)
+                                                .map(CompletableFuture::join)
+                                                .filter(Objects::nonNull)
+                                                .toList()
+                                )
+                        );
+
+        final CompletableFuture> eventStage = featuresStage.thenApply(features ->
+                FeaturesCreated.of(command.getEntityId(), features, nextRevision, getEventTimestamp(),
+                        dittoHeaders, metadata
                 )
-                .toList()
         );
 
-        final ThingEvent event =
-                FeaturesCreated.of(command.getEntityId(), features, nextRevision, getEventTimestamp(),
-                        dittoHeaders, metadata);
-        final WithDittoHeaders response = appendETagHeaderIfProvided(command,
-                ModifyFeaturesResponse.created(context.getState(), features, dittoHeaders), thing);
+        final CompletableFuture responseStage =
+                featuresStage.thenApply(features -> appendETagHeaderIfProvided(command,
+                        ModifyFeaturesResponse.created(context.getState(), features, dittoHeaders), thing)
+                );
 
-        return ResultFactory.newMutationResult(command, event, response);
+        return ResultFactory.newMutationResult(command, eventStage, responseStage);
     }
 
 
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/RetrieveFeatureStrategy.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/RetrieveFeatureStrategy.java
index f5602ea3e46..e0669d82d3a 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/RetrieveFeatureStrategy.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/RetrieveFeatureStrategy.java
@@ -13,6 +13,8 @@
 package org.eclipse.ditto.things.service.persistence.actors.strategies.commands;
 
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
 
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
@@ -20,7 +22,7 @@
 import org.eclipse.ditto.base.model.entity.metadata.Metadata;
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
-import org.eclipse.ditto.base.model.headers.DittoHeadersSettable;
+import org.eclipse.ditto.base.model.headers.WithDittoHeaders;
 import org.eclipse.ditto.base.model.headers.contenttype.ContentType;
 import org.eclipse.ditto.base.model.headers.entitytag.EntityTag;
 import org.eclipse.ditto.base.model.signals.FeatureToggle;
@@ -89,7 +91,7 @@ protected Result> doApply(final Context context,
         }
     }
 
-    private DittoHeadersSettable getRetrieveThingDescriptionResponse(@Nullable final Thing thing,
+    private CompletionStage getRetrieveThingDescriptionResponse(@Nullable final Thing thing,
             final RetrieveFeature command) {
         final String featureId = command.getFeatureId();
         if (thing != null) {
@@ -98,18 +100,23 @@ private DittoHeadersSettable getRetrieveThingDescriptionResponse(@Nullable fi
                     .map(feature -> wotThingDescriptionProvider
                             .provideFeatureTD(command.getEntityId(), thing, feature, command.getDittoHeaders())
                     )
-                    .map(td -> RetrieveWotThingDescriptionResponse.of(command.getEntityId(), td.toJson(),
-                            command.getDittoHeaders()
-                                    .toBuilder()
-                                    .contentType(ContentType.APPLICATION_TD_JSON)
-                                    .build())
+                    .map(tdStage -> tdStage.thenApply(td ->
+                            RetrieveWotThingDescriptionResponse.of(command.getEntityId(), td.toJson(),
+                                    command.getDittoHeaders()
+                                            .toBuilder()
+                                            .contentType(ContentType.APPLICATION_TD_JSON)
+                                            .build()
+                            )
+                            ).thenApply(WithDittoHeaders.class::cast)
                     )
-                    .map(DittoHeadersSettable.class::cast)
-                    .orElseGet(() -> ExceptionFactory.featureNotFound(command.getEntityId(), featureId,
-                            command.getDittoHeaders()));
+                    .orElseGet(() -> CompletableFuture.completedStage(ExceptionFactory.featureNotFound(command.getEntityId(), featureId,
+                            command.getDittoHeaders())
+                    ));
         } else {
-            return ExceptionFactory.featureNotFound(command.getEntityId(), featureId,
-                    command.getDittoHeaders());
+            return CompletableFuture.completedStage(
+                    ExceptionFactory.featureNotFound(command.getEntityId(), featureId,
+                    command.getDittoHeaders())
+            );
         }
     }
 
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/RetrieveThingStrategy.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/RetrieveThingStrategy.java
index d8e33c32441..e7f343082fb 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/RetrieveThingStrategy.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/RetrieveThingStrategy.java
@@ -14,6 +14,8 @@
 
 import java.util.Objects;
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
 
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
@@ -21,6 +23,7 @@
 import org.eclipse.ditto.base.model.entity.metadata.Metadata;
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException;
 import org.eclipse.ditto.base.model.headers.DittoHeadersSettable;
+import org.eclipse.ditto.base.model.headers.WithDittoHeaders;
 import org.eclipse.ditto.base.model.headers.contenttype.ContentType;
 import org.eclipse.ditto.base.model.headers.entitytag.EntityTag;
 import org.eclipse.ditto.base.model.signals.FeatureToggle;
@@ -39,7 +42,6 @@
 import org.eclipse.ditto.things.model.signals.commands.query.ThingQueryCommand;
 import org.eclipse.ditto.things.model.signals.events.ThingEvent;
 import org.eclipse.ditto.wot.integration.provider.WotThingDescriptionProvider;
-import org.eclipse.ditto.wot.model.ThingDescription;
 
 import akka.actor.ActorSystem;
 
@@ -122,18 +124,24 @@ private static JsonObject getThingJson(final Thing thing, final ThingQueryComman
                 .orElseGet(() -> thing.toJson(command.getImplementedSchemaVersion()));
     }
 
-    private DittoHeadersSettable getRetrieveThingDescriptionResponse(@Nullable final Thing thing,
+    private CompletionStage getRetrieveThingDescriptionResponse(@Nullable final Thing thing,
             final RetrieveThing command) {
         if (thing != null) {
-            final ThingDescription wotThingDescription = wotThingDescriptionProvider
+            return wotThingDescriptionProvider
                     .provideThingTD(thing.getDefinition().orElse(null),
                             command.getEntityId(),
                             thing,
-                            command.getDittoHeaders());
-            return RetrieveWotThingDescriptionResponse.of(command.getEntityId(), wotThingDescription.toJson(),
-                    command.getDittoHeaders().toBuilder().contentType(ContentType.APPLICATION_TD_JSON).build());
+                            command.getDittoHeaders())
+                    .thenApply(wotThingDescription ->
+                            RetrieveWotThingDescriptionResponse.of(command.getEntityId(),
+                                    wotThingDescription.toJson(),
+                                    command.getDittoHeaders().toBuilder()
+                                            .contentType(ContentType.APPLICATION_TD_JSON)
+                                            .build()
+                            )
+                    );
         } else {
-            return notAccessible(command);
+            return CompletableFuture.completedStage(notAccessible(command));
         }
     }
 
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingsConditionalHeadersValidatorProvider.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingsConditionalHeadersValidatorProvider.java
index 534061e25ee..486f12937c6 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingsConditionalHeadersValidatorProvider.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingsConditionalHeadersValidatorProvider.java
@@ -68,6 +68,13 @@ public DittoRuntimeExceptionBuilder createPreconditionNotModifiedExceptionBui
                 final String expectedNotToMatch, final String matched) {
             return ThingPreconditionNotModifiedException.newBuilder(expectedNotToMatch, matched);
         }
+
+        @Override
+        public DittoRuntimeExceptionBuilder createPreconditionNotModifiedForEqualityExceptionBuilder() {
+            return ThingPreconditionNotModifiedException.newBuilder()
+                    .message("The previous value was equal to the new value and the 'if-equal' header was set to 'skip'.")
+                    .description("Your changes were not applied, which is probably the expected outcome.");
+        }
     }
 
     private static final Set EXEMPTED_FIELDS = Collections.singleton(JsonPointer.of("_policy"));
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/events/ThingEventStrategies.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/events/ThingEventStrategies.java
index 3cf0d4e8e41..42e6ca22799 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/events/ThingEventStrategies.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/events/ThingEventStrategies.java
@@ -14,8 +14,8 @@
 
 import javax.annotation.concurrent.Immutable;
 
-import org.eclipse.ditto.things.model.Thing;
 import org.eclipse.ditto.internal.utils.persistentactors.events.AbstractEventStrategies;
+import org.eclipse.ditto.things.model.Thing;
 import org.eclipse.ditto.things.model.signals.events.AttributeCreated;
 import org.eclipse.ditto.things.model.signals.events.AttributeDeleted;
 import org.eclipse.ditto.things.model.signals.events.AttributeModified;
@@ -54,7 +54,7 @@
 import org.eclipse.ditto.things.model.signals.events.ThingModified;
 
 /**
- * This Singleton strategy handles all {@link org.eclipse.ditto.things.model.signals.events.ThingEvent}s.
+ * This Singleton strategy handles all {@link ThingEvent}s.
  */
 @Immutable
 public final class ThingEventStrategies extends AbstractEventStrategies, Thing> {
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapter.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapter.java
index 0f3e367ddfd..ba7d45a8ece 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapter.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapter.java
@@ -12,18 +12,17 @@
  */
 package org.eclipse.ditto.things.service.persistence.serializer;
 
-import javax.annotation.Nullable;
-
-import org.eclipse.ditto.base.model.json.FieldType;
-import org.eclipse.ditto.base.model.json.JsonSchemaVersion;
 import org.eclipse.ditto.base.model.signals.events.Event;
 import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry;
+import org.eclipse.ditto.base.service.config.DittoServiceConfig;
+import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig;
 import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter;
-import org.eclipse.ditto.internal.utils.persistence.mongo.DittoBsonJson;
 import org.eclipse.ditto.json.JsonObject;
+import org.eclipse.ditto.json.JsonObjectBuilder;
 import org.eclipse.ditto.json.JsonPointer;
 import org.eclipse.ditto.policies.model.Policy;
 import org.eclipse.ditto.things.model.signals.events.ThingEvent;
+import org.eclipse.ditto.things.service.common.config.DefaultThingConfig;
 
 import akka.actor.ExtendedActorSystem;
 
@@ -36,29 +35,16 @@ public final class ThingMongoEventAdapter extends AbstractMongoEventAdapter event, final JsonObject jsonObject) {
+        return super.performToJournalMigration(event, jsonObject)
                 .remove(POLICY_IN_THING_EVENT_PAYLOAD); // remove the policy entries from thing event payload
     }
 
-    @Override
-    public Object toJournal(final Object event) {
-        if (event instanceof Event theEvent) {
-            final JsonSchemaVersion schemaVersion = theEvent.getImplementedSchemaVersion();
-            final JsonObject jsonObject =
-                    theEvent.toJson(schemaVersion, FieldType.regularOrSpecial())
-                            // remove the policy entries from thing event payload
-                            .remove(POLICY_IN_THING_EVENT_PAYLOAD);
-            final DittoBsonJson dittoBsonJson = DittoBsonJson.getInstance();
-            return dittoBsonJson.parse(jsonObject);
-        } else {
-            throw new IllegalArgumentException("Unable to toJournal a non-'Event' object! Was: " + event.getClass());
-        }
-    }
-
 }
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/starter/DefaultThingPersistenceActorPropsFactory.java b/things/service/src/main/java/org/eclipse/ditto/things/service/starter/DefaultThingPersistenceActorPropsFactory.java
index 91570851eda..abca24f39fc 100644
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/starter/DefaultThingPersistenceActorPropsFactory.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/starter/DefaultThingPersistenceActorPropsFactory.java
@@ -17,6 +17,7 @@
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
 
+import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal;
 import org.eclipse.ditto.internal.utils.pubsub.DistributedPub;
 import org.eclipse.ditto.things.model.ThingId;
 import org.eclipse.ditto.things.model.signals.events.ThingEvent;
@@ -51,9 +52,10 @@ static DefaultThingPersistenceActorPropsFactory of(final ActorSystem actorSystem
     }
 
     @Override
-    public Props props(final ThingId thingId, final DistributedPub> distributedPub,
+    public Props props(final ThingId thingId, final MongoReadJournal mongoReadJournal,
+            final DistributedPub> distributedPub,
             @Nullable final ActorRef searchShardRegionProxy) {
         argumentNotEmpty(thingId);
-        return ThingPersistenceActor.props(thingId, distributedPub, searchShardRegionProxy);
+        return ThingPersistenceActor.props(thingId, mongoReadJournal, distributedPub, searchShardRegionProxy);
     }
 }
diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/starter/ThingsRootActor.java b/things/service/src/main/java/org/eclipse/ditto/things/service/starter/ThingsRootActor.java
index 3c6fe00e7c4..83dbd50e29a 100755
--- a/things/service/src/main/java/org/eclipse/ditto/things/service/starter/ThingsRootActor.java
+++ b/things/service/src/main/java/org/eclipse/ditto/things/service/starter/ThingsRootActor.java
@@ -88,12 +88,14 @@ private ThingsRootActor(final ThingsConfig thingsConfig,
 
         final BlockedNamespaces blockedNamespaces = BlockedNamespaces.of(actorSystem);
         final PolicyEnforcerProvider policyEnforcerProvider = PolicyEnforcerProviderExtension.get(actorSystem).getPolicyEnforcerProvider();
+        final var mongoReadJournal = newMongoReadJournal(thingsConfig.getMongoDbConfig(), actorSystem);
         final Props thingSupervisorActorProps = getThingSupervisorActorProps(pubSubMediator,
                 distributedPubThingEventsForTwin,
                 liveSignalPub,
                 propsFactory,
                 blockedNamespaces,
-                policyEnforcerProvider
+                policyEnforcerProvider,
+                mongoReadJournal
         );
 
         final ActorRef thingsShardRegion =
@@ -128,10 +130,9 @@ private ThingsRootActor(final ThingsConfig thingsConfig,
                 DefaultHealthCheckingActorFactory.props(healthCheckingActorOptions, MongoHealthChecker.props()));
 
         final ActorRef snapshotStreamingActor =
-                ThingsPersistenceStreamingActorCreator.startSnapshotStreamingActor(this::startChildActor);
+                ThingsPersistenceStreamingActorCreator.startPersistenceStreamingActor(this::startChildActor);
 
         final var cleanupConfig = thingsConfig.getThingConfig().getCleanupConfig();
-        final var mongoReadJournal = newMongoReadJournal(thingsConfig.getMongoDbConfig(), actorSystem);
         final Props cleanupActorProps = PersistenceCleanupActor.props(cleanupConfig, mongoReadJournal, CLUSTER_ROLE);
         startChildActor(PersistenceCleanupActor.ACTOR_NAME, cleanupActorProps);
 
@@ -174,9 +175,10 @@ private static Props getThingSupervisorActorProps(final ActorRef pubSubMediator,
             final LiveSignalPub liveSignalPub,
             final ThingPersistenceActorPropsFactory propsFactory,
             final BlockedNamespaces blockedNamespaces,
-            final PolicyEnforcerProvider policyEnforcerProvider) {
+            final PolicyEnforcerProvider policyEnforcerProvider,
+            final MongoReadJournal mongoReadJournal) {
         return ThingSupervisorActor.props(pubSubMediator, distributedPubThingEventsForTwin,
-                liveSignalPub, propsFactory, blockedNamespaces, policyEnforcerProvider);
+                liveSignalPub, propsFactory, blockedNamespaces, policyEnforcerProvider, mongoReadJournal);
     }
 
     private static MongoReadJournal newMongoReadJournal(final MongoDbConfig mongoDbConfig,
diff --git a/things/service/src/main/resources/things.conf b/things/service/src/main/resources/things.conf
index dd6dcf944d7..b003e2d3329 100755
--- a/things/service/src/main/resources/things.conf
+++ b/things/service/src/main/resources/things.conf
@@ -9,7 +9,7 @@ ditto {
       "org.eclipse.ditto.policies.enforcement.pre.CreationRestrictionPreEnforcer"
     ]
     signal-transformers-provider.extension-config.signal-transformers = [
-      "org.eclipse.ditto.things.service.enforcement.pre.ModifyToCreateThingTransformer", // always keep this as first transformer in order to guarantee that all following transformers know that the command is creating a policy instead of modifying it
+      "org.eclipse.ditto.things.service.enforcement.pre.ModifyToCreateThingTransformer", // always keep this as first transformer in order to guarantee that all following transformers know that the command is creating a thing instead of modifying it
       "org.eclipse.ditto.things.service.signaltransformation.placeholdersubstitution.ThingsPlaceholderSubstitution"
     ]
     snapshot-adapter = "org.eclipse.ditto.things.service.persistence.serializer.ThingMongoSnapshotAdapter"
@@ -54,6 +54,19 @@ ditto {
         threshold = ${?THING_SNAPSHOT_THRESHOLD} # may be overridden with this environment variable
       }
 
+      event {
+        # define the DittoHeaders to persist when persisting events to the journal
+        # those can e.g. be retrieved as additional "audit log" information when accessing a historical thing revision
+        historical-headers-to-persist = [
+          #"ditto-originator"  # who (user-subject/connection-pre-auth-subject) issued the event
+          #"correlation-id"
+          #"ditto-origin"      # which WS session or connection issued the event
+          #"origin"            # the HTTP origin header
+          #"user-agent"        # the HTTP user-agent header
+        ]
+        historical-headers-to-persist = ${?THING_EVENT_HISTORICAL_HEADERS_TO_PERSIST}
+      }
+
       supervisor {
         exponential-backoff {
           min = 1s
@@ -61,30 +74,70 @@ ditto {
           random-factor = 1.0
           corrupted-receive-timeout = 600s
         }
+
+        local-ask {
+          timeout = 5s
+          timeout = ${?THINGS_SUPERVISOR_LOCAL_ASK_TIMEOUT}
+        }
       }
 
       cleanup {
+        # enabled configures whether background cleanup is enabled or not
+        # If enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process
         enabled = true
         enabled = ${?CLEANUP_ENABLED}
 
+        # history-retention-duration configures the duration of how long to "keep" events and snapshots before being
+        # allowed to remove them in scope of cleanup.
+        # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read
+        # journal.
+        history-retention-duration = 3d
+        history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION}
+
+        # quiet-period defines how long to stay in a state where the background cleanup is not yet started
+        # Applies after:
+        # - starting the service
+        # - each "completed" background cleanup run (all entities were cleaned up)
         quiet-period = 5m
         quiet-period = ${?CLEANUP_QUIET_PERIOD}
 
+        # interval configures how often a "credit decision" is made.
+        # The background cleanup works with a credit system and does only generate new "cleanup credits" if the MongoDB
+        # currently has capacity to do cleanups.
         interval = 3s
         interval = ${?CLEANUP_INTERVAL}
 
+        # timer-threshold configures the maximum database latency to give out credit for cleanup actions.
+        # If write operations to the MongoDB within the last `interval` had a `max` value greater to the configured
+        # threshold, no new cleanup credits will be issued for the next `interval`.
+        # Which throttles cleanup when MongoDB is currently under heavy (write) load.
         timer-threshold = 150ms
         timer-threshold = ${?CLEANUP_TIMER_THRESHOLD}
 
+        # credits-per-batch configures how many "cleanup credits" should be generated per `interval` as long as the
+        # write operations to the MongoDB are less than the configured `timer-threshold`.
+        # Limits the rate of cleanup actions to this many per credit decision interval.
+        # One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`.
         credits-per-batch = 3
         credits-per-batch = ${?CLEANUP_CREDITS_PER_BATCH}
 
+        # reads-per-query configures the number of snapshots to scan per MongoDB query.
+        # Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned
+        # with the amount of "cleanup credits" issued per `interval` - in order to avoid long running queries.
         reads-per-query = 100
         reads-per-query = ${?CLEANUP_READS_PER_QUERY}
 
+        # writes-per-credit configures the number of documents to delete for each credit.
+        # If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead
+        # to 10 delete operations performed against MongoDB.
         writes-per-credit = 100
         writes-per-credit = ${?CLEANUP_WRITES_PER_CREDIT}
 
+        # delete-final-deleted-snapshot configures whether for a deleted entity, the final snapshot (containing the
+        # "deleted" information) should be deleted or not.
+        # If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with
+        # a revision number 1 higher than the previously deleted entity. If the final snapshot is deleted as well,
+        # recreation of an entity with the same ID will lead to revisionNumber=1 after its recreation.
         delete-final-deleted-snapshot = false
         delete-final-deleted-snapshot = ${?CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT}
       }
@@ -132,7 +185,7 @@ ditto {
         #     }
         #   },
         #   "security": "oauth2_google_sc"
-        #   "support": "https://www.eclipse.org/ditto/"
+        #   "support": "https://www.eclipse.dev/ditto/"
         # }
 
         json-template {
@@ -143,7 +196,7 @@ ditto {
             }
           },
           "security": "basic_sc"
-          "support": "https://www.eclipse.org/ditto/"
+          "support": "https://www.eclipse.dev/ditto/"
         }
 
         placeholders {
@@ -229,6 +282,18 @@ akka-contrib-mongodb-persistence-things-journal {
   }
 }
 
+akka-contrib-mongodb-persistence-things-journal-read {
+  class = "akka.contrib.persistence.mongodb.MongoReadJournal"
+  plugin-dispatcher = "thing-journal-persistence-dispatcher"
+
+  overrides {
+    journal-collection = "things_journal"
+    journal-index = "things_journal_index"
+    realtime-collection = "things_realtime"
+    metadata-collection = "things_metadata"
+  }
+}
+
 akka-contrib-mongodb-persistence-things-snapshots {
   class = "akka.contrib.persistence.mongodb.MongoSnapshots"
   plugin-dispatcher = "thing-snaps-persistence-dispatcher"
@@ -284,6 +349,10 @@ wot-dispatcher {
   type = Dispatcher
   executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
 }
+wot-dispatcher-cache-loader {
+  type = Dispatcher
+  executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
+}
 
 blocked-namespaces-dispatcher {
   type = Dispatcher
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/AbstractThingEnforcementTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/AbstractThingEnforcementTest.java
index 5558cc9308a..b658894c2b6 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/AbstractThingEnforcementTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/AbstractThingEnforcementTest.java
@@ -24,6 +24,7 @@
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.ditto.base.model.auth.AuthorizationSubject;
+import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal;
 import org.eclipse.ditto.internal.utils.pubsub.DistributedPub;
 import org.eclipse.ditto.internal.utils.pubsub.extractors.AckExtractor;
 import org.eclipse.ditto.policies.enforcement.PolicyEnforcerProvider;
@@ -127,7 +128,8 @@ public > Object wrapForPublicationWithAcks(final S messa
                 new TestSetup.DummyLiveSignalPub(pubSubMediatorProbe.ref()),
                 thingPersistenceActorProbe.ref(),
                 null,
-                policyEnforcerProvider
+                policyEnforcerProvider,
+                Mockito.mock(MongoReadJournal.class)
         ).withDispatcher("akka.actor.default-dispatcher"), system.guardian(),
                 URLEncoder.encode(THING_ID.toString(), Charset.defaultCharset()));
         // Actors using "stash()" require the above dispatcher to be configured, otherwise stash() and unstashAll() won't
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/MergeThingCommandEnforcementTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/MergeThingCommandEnforcementTest.java
index e1dec6f3477..29d6068b885 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/MergeThingCommandEnforcementTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/MergeThingCommandEnforcementTest.java
@@ -32,6 +32,7 @@
 import org.eclipse.ditto.json.JsonFactory;
 import org.eclipse.ditto.json.JsonObject;
 import org.eclipse.ditto.json.JsonPointer;
+import org.eclipse.ditto.json.JsonValue;
 import org.eclipse.ditto.policies.model.EffectedPermissions;
 import org.eclipse.ditto.policies.model.PoliciesModelFactory;
 import org.eclipse.ditto.policies.model.Policy;
@@ -41,6 +42,7 @@
 import org.eclipse.ditto.things.api.Permission;
 import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingNotModifiableException;
 import org.eclipse.ditto.things.model.signals.commands.modify.MergeThing;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
@@ -119,6 +121,44 @@ void rejectByPolicy(final TestArgument arg) {
                 () -> ThingCommandEnforcement.authorizeByPolicyOrThrow(policyEnforcer, arg.getMergeThing()));
     }
 
+    @Test
+    void acceptUsingRegexInPolicy() {
+        final TestArgument testArgument = TestArgument.of("/",
+                Set.of("/", "/attributes/complex"),
+                Set.of("/attributes/simple")
+        );
+        final TrieBasedPolicyEnforcer policyEnforcer = TrieBasedPolicyEnforcer.newInstance(testArgument.getPolicy());
+        final JsonObject patch = JsonObject.newBuilder()
+                .set("attributes", JsonObject.newBuilder()
+                        .set("complex", JsonObject.newBuilder()
+                                .set("{{ /.*/ }}", JsonValue.nullLiteral())
+                                .build()
+                        )
+                        .set("another", 42)
+                        .build())
+                .build();
+        final MergeThing mergeThing = MergeThing.of(TestSetup.THING_ID, JsonPointer.empty(), patch, TestArgument.headers());
+        final MergeThing authorizedMergeThing =
+                ThingCommandEnforcement.authorizeByPolicyOrThrow(policyEnforcer, mergeThing);
+        assertThat(authorizedMergeThing.getDittoHeaders().getAuthorizationContext()).isNotNull();
+    }
+
+    @Test
+    void rejectUsingRegexWithRevokesInPolicy() {
+        final TestArgument testArgument = TestArgument.of("/", Set.of("/"), Set.of("/attributes/complex/nested/secret"));
+        final TrieBasedPolicyEnforcer policyEnforcer = TrieBasedPolicyEnforcer.newInstance(testArgument.getPolicy());
+        final JsonObject patch = JsonObject.newBuilder()
+                .set("attributes", JsonObject.newBuilder()
+                        .set("{{ /.*/ }}", JsonValue.nullLiteral())
+                        .set("simple", "value")
+                        .set("another", 42)
+                        .build())
+                .build();
+        final MergeThing mergeThing = MergeThing.of(TestSetup.THING_ID, JsonPointer.empty(), patch, TestArgument.headers());
+        assertThatExceptionOfType(ThingNotModifiableException.class).isThrownBy(
+                () -> ThingCommandEnforcement.authorizeByPolicyOrThrow(policyEnforcer, mergeThing));
+    }
+
     /**
      * Generates combinations of a Policy and MergeThing commands that should be accepted by command enforcement.
      */
@@ -244,7 +284,7 @@ private static MergeThing toMergeCommand(final JsonPointer path, final JsonObjec
             return MergeThing.of(TestSetup.THING_ID, path, patch.getValue(path).orElseThrow(), headers());
         }
 
-        private static DittoHeaders headers() {
+        static DittoHeaders headers() {
             return DittoHeaders.newBuilder()
                     .authorizationContext(
                             AuthorizationContext.newInstance(DittoAuthorizationContextType.UNSPECIFIED,
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/MultiStageCommandEnforcementTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/MultiStageCommandEnforcementTest.java
index 44f3a4686f9..df7facea575 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/MultiStageCommandEnforcementTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/MultiStageCommandEnforcementTest.java
@@ -47,6 +47,8 @@
 import org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyNotAccessibleException;
 import org.eclipse.ditto.policies.model.signals.commands.modify.CreatePolicy;
 import org.eclipse.ditto.policies.model.signals.commands.modify.CreatePolicyResponse;
+import org.eclipse.ditto.policies.model.signals.commands.modify.DeletePolicy;
+import org.eclipse.ditto.policies.model.signals.commands.modify.DeletePolicyResponse;
 import org.eclipse.ditto.policies.model.signals.commands.query.RetrievePolicy;
 import org.eclipse.ditto.policies.model.signals.commands.query.RetrievePolicyResponse;
 import org.eclipse.ditto.things.api.commands.sudo.SudoRetrieveThingResponse;
@@ -450,6 +452,8 @@ public void createThingWithExplicitPolicyNotAuthorizedBySelf() {
 
             thingPersistenceActorProbe.expectMsgClass(CreateThing.class);
             thingPersistenceActorProbe.reply(ThingNotModifiableException.newBuilder(thingId).build());
+            policiesShardRegionProbe.expectMsgClass(DeletePolicy.class);
+            policiesShardRegionProbe.reply(DeletePolicyResponse.of(policyId, DEFAULT_HEADERS));
 
             // THEN: initial requester receives error
             expectMsgClass(ThingNotModifiableException.class);
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/pre/ModifyToCreateThingTransformerTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/pre/ModifyToCreateThingTransformerTest.java
index 27623c153cd..b3eb91a702b 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/pre/ModifyToCreateThingTransformerTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/pre/ModifyToCreateThingTransformerTest.java
@@ -24,7 +24,9 @@
 import org.eclipse.ditto.base.model.signals.Signal;
 import org.eclipse.ditto.things.model.Thing;
 import org.eclipse.ditto.things.model.ThingId;
+import org.eclipse.ditto.things.model.ThingsModelFactory;
 import org.eclipse.ditto.things.model.signals.commands.modify.CreateThing;
+import org.eclipse.ditto.things.model.signals.commands.modify.MergeThing;
 import org.eclipse.ditto.things.model.signals.commands.modify.ModifyThing;
 import org.eclipse.ditto.things.model.signals.commands.query.RetrieveThing;
 import org.junit.Before;
@@ -58,6 +60,36 @@ public void modifyThingStaysModifyThingWhenAlreadyExisting() {
         verify(existenceChecker).checkExistence(modifyThing);
     }
 
+    @Test
+    public void mergeThingBecomesCreateThingPolicyWhenNotYetExisting() {
+        final var thingId = ThingId.generateRandom();
+        final var mergeThing = MergeThing.withThing(thingId, Thing.newBuilder().setId(thingId).build(),
+                DittoHeaders.of(Map.of("foo", "bar")));
+        when(existenceChecker.checkExistence(mergeThing)).thenReturn(CompletableFuture.completedStage(false));
+
+        final Signal result = underTest.apply(mergeThing).toCompletableFuture().join();
+
+        assertThat(result).isInstanceOf(CreateThing.class);
+        final CreateThing createThing = (CreateThing) result;
+        assertThat(createThing.getEntityId().toString()).hasToString(thingId.toString());
+        assertThat(createThing.getThing()).isEqualTo(ThingsModelFactory.newThing(mergeThing.getValue().asObject()));
+        assertThat(createThing.getDittoHeaders()).isSameAs(mergeThing.getDittoHeaders());
+        verify(existenceChecker).checkExistence(mergeThing);
+    }
+
+    @Test
+    public void mergeThingStaysMergeThingWhenAlreadyExisting() {
+        final var thingId = ThingId.generateRandom();
+        final var mergeThing = MergeThing.withThing(thingId, Thing.newBuilder().setId(thingId).build(),
+                DittoHeaders.of(Map.of("foo", "bar")));
+        when(existenceChecker.checkExistence(mergeThing)).thenReturn(CompletableFuture.completedStage(true));
+
+        final Signal result = underTest.apply(mergeThing).toCompletableFuture().join();
+
+        assertThat(result).isSameAs(mergeThing);
+        verify(existenceChecker).checkExistence(mergeThing);
+    }
+
     @Test
     public void modifyThingBecomesCreateThingPolicyWhenNotYetExisting() {
         final var thingId = ThingId.generateRandom();
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/PersistenceActorTestBase.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/PersistenceActorTestBase.java
index 7068f4f7eef..fc1473de9a6 100755
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/PersistenceActorTestBase.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/PersistenceActorTestBase.java
@@ -28,6 +28,7 @@
 import org.eclipse.ditto.base.model.entity.metadata.Metadata;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.base.model.json.JsonSchemaVersion;
+import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal;
 import org.eclipse.ditto.internal.utils.pubsub.DistributedPub;
 import org.eclipse.ditto.internal.utils.pubsub.extractors.AckExtractor;
 import org.eclipse.ditto.internal.utils.pubsubthings.LiveSignalPub;
@@ -226,11 +227,13 @@ protected ActorRef createPersistenceActorFor(final ThingId thingId) {
 
     protected ActorRef createPersistenceActorWithPubSubFor(final ThingId thingId) {
 
-        return actorSystem.actorOf(getPropsOfThingPersistenceActor(thingId, getDistributedPub()));
+        return actorSystem.actorOf(getPropsOfThingPersistenceActor(thingId, Mockito.mock(MongoReadJournal.class),
+                getDistributedPub()));
     }
 
-    private Props getPropsOfThingPersistenceActor(final ThingId thingId, final DistributedPub> pub) {
-        return ThingPersistenceActor.props(thingId, pub, null);
+    private Props getPropsOfThingPersistenceActor(final ThingId thingId, final MongoReadJournal mongoReadJournal,
+            final DistributedPub> pub) {
+        return ThingPersistenceActor.props(thingId, mongoReadJournal, pub, null);
     }
 
     protected ActorRef createSupervisorActorFor(final ThingId thingId) {
@@ -258,9 +261,11 @@ public > Object wrapForPublicationWithAcks(final S messa
                             }
                         },
                         liveSignalPub,
-                        (thingId1, pub, searchShardRegionProxy) -> getPropsOfThingPersistenceActor(thingId1, pub),
+                        (thingId1, mongoReadJournal, pub, searchShardRegionProxy) -> getPropsOfThingPersistenceActor(
+                                thingId1, mongoReadJournal, pub),
                         null,
-                        policyEnforcerProvider);
+                        policyEnforcerProvider,
+                        Mockito.mock(MongoReadJournal.class));
 
         return actorSystem.actorOf(props, thingId.toString());
     }
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorSnapshottingTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorSnapshottingTest.java
index 0f4d64fed6c..69751dab4c1 100755
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorSnapshottingTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorSnapshottingTest.java
@@ -16,8 +16,10 @@
 import java.util.Arrays;
 import java.util.Collections;
 
+import org.assertj.core.api.Assertions;
 import org.eclipse.ditto.base.model.common.HttpStatus;
 import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent;
+import org.eclipse.ditto.internal.utils.config.DittoConfigError;
 import org.eclipse.ditto.internal.utils.test.Retry;
 import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource;
 import org.eclipse.ditto.json.JsonFactory;
@@ -233,13 +235,7 @@ public void thingInArbitraryStateIsSnapshotCorrectly() {
     public void actorCannotBeStartedWithNegativeSnapshotThreshold() {
         final Config customConfig = createNewDefaultTestConfig().withValue(SNAPSHOT_THRESHOLD,
                 ConfigValueFactory.fromAnyRef(-1));
-        setup(customConfig);
-
-        disableLogging();
-        new TestKit(actorSystem) {{
-            final ActorRef underTest = createPersistenceActorFor(ThingId.of("fail:fail"));
-            watch(underTest);
-            expectTerminated(underTest);
-        }};
+        Assertions.assertThatExceptionOfType(DittoConfigError.class)
+                        .isThrownBy(() -> setup(customConfig));
     }
 }
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorTest.java
index 533a758568d..d338ad604fa 100755
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorTest.java
@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Optional;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
@@ -44,6 +45,10 @@
 import org.eclipse.ditto.base.model.json.JsonSchemaVersion;
 import org.eclipse.ditto.base.model.signals.events.Event;
 import org.eclipse.ditto.internal.utils.cluster.DistPubSubAccess;
+import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal;
+import org.eclipse.ditto.internal.utils.pubsub.DistributedPub;
+import org.eclipse.ditto.internal.utils.pubsub.extractors.AckExtractor;
+import org.eclipse.ditto.internal.utils.pubsubthings.LiveSignalPub;
 import org.eclipse.ditto.internal.utils.test.Retry;
 import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource;
 import org.eclipse.ditto.json.JsonFactory;
@@ -62,6 +67,8 @@
 import org.eclipse.ditto.policies.model.SubjectIssuer;
 import org.eclipse.ditto.policies.model.signals.commands.modify.CreatePolicy;
 import org.eclipse.ditto.policies.model.signals.commands.modify.CreatePolicyResponse;
+import org.eclipse.ditto.policies.model.signals.commands.modify.DeletePolicy;
+import org.eclipse.ditto.policies.model.signals.commands.modify.DeletePolicyResponse;
 import org.eclipse.ditto.things.api.Permission;
 import org.eclipse.ditto.things.model.Attributes;
 import org.eclipse.ditto.things.model.Feature;
@@ -111,23 +118,28 @@
 import org.eclipse.ditto.things.model.signals.events.ThingCreated;
 import org.eclipse.ditto.things.model.signals.events.ThingEvent;
 import org.eclipse.ditto.things.model.signals.events.ThingModified;
+import org.eclipse.ditto.things.service.enforcement.TestSetup;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestWatcher;
+import org.mockito.Mockito;
 import org.slf4j.LoggerFactory;
 
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import com.typesafe.config.ConfigValueFactory;
 
+import akka.actor.AbstractActor;
 import akka.actor.ActorRef;
 import akka.actor.ActorSelection;
 import akka.actor.PoisonPill;
 import akka.actor.Props;
 import akka.cluster.pubsub.DistributedPubSubMediator;
+import akka.japi.pf.ReceiveBuilder;
 import akka.testkit.TestActorRef;
+import akka.testkit.TestProbe;
 import akka.testkit.javadsl.TestKit;
 import scala.PartialFunction;
 import scala.concurrent.Await;
@@ -304,7 +316,8 @@ public void tryToCreateThingWithDifferentThingId() {
         final Thing thing = createThingV2WithRandomId();
         final CreateThing createThing = CreateThing.of(thing, null, dittoHeadersV2);
 
-        final Props props = ThingPersistenceActor.props(thingIdOfActor, getDistributedPub(), null);
+        final Props props = ThingPersistenceActor.props(thingIdOfActor, Mockito.mock(MongoReadJournal.class),
+                getDistributedPub(), null);
         final TestActorRef underTest = TestActorRef.create(actorSystem, props);
         final ThingPersistenceActor thingPersistenceActor = underTest.underlyingActor();
         final PartialFunction receiveCommand = thingPersistenceActor.receiveCommand();
@@ -2001,6 +2014,227 @@ public void testRemovalOfEmptyMetadataAfterDeletion() {
         }};
     }
 
+    @Test
+    public void unavailableExpectedAndPolicyIsDeletedIfPersistenceActorFails() {
+        final DittoHeaders dittoHeaders = dittoHeadersV2.toBuilder()
+                .correlationId(UUID.randomUUID().toString())
+                .build();
+        final Policy inlinePolicy = PoliciesModelFactory.newPolicyBuilder(POLICY_ID)
+                .setRevision(1L)
+                .forLabel("authorize-self")
+                .setSubject(SubjectIssuer.newInstance("test"), AUTH_SUBJECT)
+                .setGrantedPermissions(PoliciesResourceType.thingResource(JsonPointer.empty()),
+                        Permissions.newInstance(Permission.READ, Permission.WRITE))
+                .setGrantedPermissions(PoliciesResourceType.policyResource(JsonPointer.empty()),
+                        Permissions.newInstance(Permission.READ, Permission.WRITE))
+                .setGrantedPermissions(PoliciesResourceType.messageResource(JsonPointer.empty()),
+                        Permissions.newInstance(Permission.READ, Permission.WRITE))
+                .build();
+        final CreatePolicyResponse createPolicyResponse = CreatePolicyResponse.of(POLICY_ID, inlinePolicy,
+                DittoHeaders.empty());
+        when(policyEnforcerProvider.getPolicyEnforcer(POLICY_ID))
+                .thenReturn(CompletableFuture.completedStage(Optional.of(PolicyEnforcer.of(inlinePolicy))));
+
+        final DeletePolicyResponse deletePolicyResponse = DeletePolicyResponse.of(POLICY_ID, dittoHeaders);
+
+        new TestKit(actorSystem) {
+            {
+                Thing thing = createThingV2WithRandomId().toBuilder().setPolicyId(null).build();
+                ThingId thingId = getIdOrThrow(thing);
+
+                ActorRef underTest = createSupervisorActorWithCustomPersistenceActor(thingId,
+                        (thingId1, mongoReadJournal, distributedPub, searchShardRegionProxy) -> FailingInCtorActor.props());
+
+                CreateThing createThing = CreateThing.of(thing, null, dittoHeaders);
+                underTest.tell(createThing, getRef());
+                policiesShardRegionTestProbe.expectMsgClass(CreatePolicy.class);
+                policiesShardRegionTestProbe.reply(createPolicyResponse);
+                policiesShardRegionTestProbe.expectMsgClass(DeletePolicy.class);
+                policiesShardRegionTestProbe.reply(deletePolicyResponse);
+                expectMsgClass(ThingUnavailableException.class);
+                expectNoMessage();
+
+            }
+        };
+    }
+
+    @Test
+    public void policyShouldNotBeDeletedOnThingRetrieveAndActorFail() {
+        final DittoHeaders dittoHeaders = dittoHeadersV2.toBuilder()
+                .correlationId(UUID.randomUUID().toString())
+                .build();
+        final Policy inlinePolicy = PoliciesModelFactory.newPolicyBuilder(POLICY_ID)
+                .setRevision(1L)
+                .forLabel("authorize-self")
+                .setSubject(SubjectIssuer.newInstance("test"), AUTH_SUBJECT)
+                .setGrantedPermissions(PoliciesResourceType.thingResource(JsonPointer.empty()),
+                        Permissions.newInstance(Permission.READ, Permission.WRITE))
+                .setGrantedPermissions(PoliciesResourceType.policyResource(JsonPointer.empty()),
+                        Permissions.newInstance(Permission.READ, Permission.WRITE))
+                .setGrantedPermissions(PoliciesResourceType.messageResource(JsonPointer.empty()),
+                        Permissions.newInstance(Permission.READ, Permission.WRITE))
+                .build();
+        when(policyEnforcerProvider.getPolicyEnforcer(POLICY_ID))
+                .thenReturn(CompletableFuture.completedStage(Optional.of(PolicyEnforcer.of(inlinePolicy))));
+
+        new TestKit(actorSystem) {
+            {
+                Thing thing = createThingV2WithRandomId().toBuilder().setPolicyId(null).build();
+                ThingId thingId = getIdOrThrow(thing);
+
+                ActorRef underTest = createSupervisorActorWithCustomPersistenceActor(thingId,
+                        (thingId1, mongoReadJournal, distributedPub, searchShardRegionProxy) -> FailingInCtorActor.props());
+
+                RetrieveThing retrieveThing = RetrieveThing.of(thingId, dittoHeaders);
+                underTest.tell(retrieveThing, getRef());
+                policiesShardRegionTestProbe.expectNoMessage();
+                expectMsgClass(ThingUnavailableException.class);
+                expectNoMessage();
+
+            }
+        };
+    }
+
+    @Test
+    public void policyShouldBeDeletedOnThingCreateAndErrorResponse() {
+        final DittoHeaders dittoHeaders = dittoHeadersV2.toBuilder()
+                .correlationId(UUID.randomUUID().toString())
+                .build();
+        final Policy inlinePolicy = PoliciesModelFactory.newPolicyBuilder(POLICY_ID)
+                .setRevision(1L)
+                .forLabel("authorize-self")
+                .setSubject(SubjectIssuer.newInstance("test"), AUTH_SUBJECT)
+                .setGrantedPermissions(PoliciesResourceType.thingResource(JsonPointer.empty()),
+                        Permissions.newInstance(Permission.READ, Permission.WRITE))
+                .setGrantedPermissions(PoliciesResourceType.policyResource(JsonPointer.empty()),
+                        Permissions.newInstance(Permission.READ, Permission.WRITE))
+                .setGrantedPermissions(PoliciesResourceType.messageResource(JsonPointer.empty()),
+                        Permissions.newInstance(Permission.READ, Permission.WRITE))
+                .build();
+        final CreatePolicyResponse createPolicyResponse = CreatePolicyResponse.of(POLICY_ID, inlinePolicy,
+                DittoHeaders.empty());
+        when(policyEnforcerProvider.getPolicyEnforcer(POLICY_ID))
+                .thenReturn(CompletableFuture.completedStage(Optional.of(PolicyEnforcer.of(inlinePolicy))));
+
+        final DeletePolicyResponse deletePolicyResponse = DeletePolicyResponse.of(POLICY_ID, dittoHeaders);
+
+        new TestKit(actorSystem) {
+            {
+                Thing thing = createThingV2WithRandomId().toBuilder().setPolicyId(null).build();
+                ThingId thingId = getIdOrThrow(thing);
+
+                TestProbe testProbe = TestProbe.apply("mock-thingPersistenceActorProbe", actorSystem);
+                ActorRef underTest = createSupervisorActorWithCustomPersistenceActor(thingId, testProbe.ref());
+
+                CreateThing createThing = CreateThing.of(thing, null, dittoHeaders);
+                underTest.tell(createThing, getRef());
+
+                testProbe.expectNoMsg();
+
+
+                policiesShardRegionTestProbe.expectMsgClass(CreatePolicy.class);
+                policiesShardRegionTestProbe.reply(createPolicyResponse);
+
+                testProbe.expectMsgClass(CreateThing.class);
+                testProbe.reply(ThingUnavailableException.newBuilder(thingId)
+                        .dittoHeaders(dittoHeaders)
+                        .message("Error in target persistent actor").build());
+
+                policiesShardRegionTestProbe.expectMsgClass(DeletePolicy.class);
+                policiesShardRegionTestProbe.reply(deletePolicyResponse);
+                expectMsgClass(ThingUnavailableException.class);
+                expectNoMessage();
+
+            }
+        };
+    }
+
+    public static final class FailingInCtorActor extends AbstractActor {
+
+        public FailingInCtorActor() {
+            super();
+            throw new RuntimeException("Failed to create PersistenceActor");
+        }
+
+        @Override
+        public Receive createReceive() {
+            return ReceiveBuilder.create().build();
+        }
+
+        private static Props props() {
+            return Props.create(FailingInCtorActor.class);
+        }
+    }
+
+    private ActorRef createSupervisorActorWithCustomPersistenceActor(final ThingId thingId,
+            final ThingPersistenceActorPropsFactory persistenceActorPropsFactory) {
+        final LiveSignalPub liveSignalPub = new TestSetup.DummyLiveSignalPub(pubSubMediator);
+        final Props props =
+                ThingSupervisorActor.props(pubSubMediator,
+                        policiesShardRegion,
+                        new DistributedPub<>() {
+
+                            @Override
+                            public ActorRef getPublisher() {
+                                return pubSubMediator;
+                            }
+
+                            @Override
+                            public Object wrapForPublication(final ThingEvent message,
+                                    final CharSequence groupIndexKey) {
+                                return message;
+                            }
+
+                            @Override
+                            public > Object wrapForPublicationWithAcks(final S message,
+                                    final CharSequence groupIndexKey, final AckExtractor ackExtractor) {
+                                return wrapForPublication(message, groupIndexKey);
+                            }
+                        },
+                        liveSignalPub,
+                        persistenceActorPropsFactory,
+                        null,
+                        policyEnforcerProvider,
+                        Mockito.mock(MongoReadJournal.class));
+
+        return actorSystem.actorOf(props, thingId.toString());
+    }
+
+    private ActorRef createSupervisorActorWithCustomPersistenceActor(final ThingId thingId,
+            final ActorRef persistenceActor) {
+        final LiveSignalPub liveSignalPub = new TestSetup.DummyLiveSignalPub(pubSubMediator);
+        final Props props =
+                ThingSupervisorActor.props(pubSubMediator,
+                        policiesShardRegion,
+                        new DistributedPub<>() {
+
+                            @Override
+                            public ActorRef getPublisher() {
+                                return pubSubMediator;
+                            }
+
+                            @Override
+                            public Object wrapForPublication(final ThingEvent message,
+                                    final CharSequence groupIndexKey) {
+                                return message;
+                            }
+
+                            @Override
+                            public > Object wrapForPublicationWithAcks(final S message,
+                                    final CharSequence groupIndexKey, final AckExtractor ackExtractor) {
+                                return wrapForPublication(message, groupIndexKey);
+                            }
+                        },
+                        liveSignalPub,
+                        persistenceActor,
+                        null,
+                        policyEnforcerProvider,
+                        Mockito.mock(MongoReadJournal.class));
+
+        return actorSystem.actorOf(props, thingId.toString());
+    }
+
+
     private void assertPublishEvent(final ThingEvent event) {
         final ThingEvent msg = pubSubTestProbe.expectMsgClass(ThingEvent.class);
         Assertions.assertThat(msg.toJson())
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceOperationsActorIT.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceOperationsActorIT.java
index f414733180a..5316bb94124 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceOperationsActorIT.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceOperationsActorIT.java
@@ -15,6 +15,7 @@
 import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.internal.utils.persistence.mongo.ops.eventsource.MongoEventSourceITAssertions;
+import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal;
 import org.eclipse.ditto.internal.utils.pubsub.DistributedPub;
 import org.eclipse.ditto.internal.utils.pubsub.extractors.AckExtractor;
 import org.eclipse.ditto.internal.utils.pubsubthings.LiveSignalPub;
@@ -149,10 +150,15 @@ public > Object wrapForPublicationWithAcks(final S messa
                     }
                 },
                 liveSignalPub,
-                (thingId, distributedPub, searchShardRegionProxy) -> ThingPersistenceActor.props(thingId,
-                        distributedPub, null),
+                (thingId, mongoReadJournal, distributedPub, searchShardRegionProxy) -> ThingPersistenceActor.props(
+                        thingId,
+                        mongoReadJournal,
+                        distributedPub,
+                        null
+                ),
                 null,
-                policyEnforcerProvider);
+                policyEnforcerProvider,
+                Mockito.mock(MongoReadJournal.class));
 
         return system.actorOf(props, id.toString());
     }
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/AbstractCommandStrategyTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/AbstractCommandStrategyTest.java
index 2e27c326e92..17fd6848ea5 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/AbstractCommandStrategyTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/AbstractCommandStrategyTest.java
@@ -20,10 +20,12 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
+import java.util.concurrent.CompletionStage;
 import java.util.function.Consumer;
 
 import javax.annotation.Nullable;
 
+import org.assertj.core.api.CompletableFutureAssert;
 import org.eclipse.ditto.base.model.common.DittoSystemProperties;
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
@@ -97,6 +99,20 @@ protected static , T extends ThingModifiedEvent> T asser
         return assertModificationResult(result, expectedEventClass, expectedCommandResponse, becomeDeleted);
     }
 
+    protected static , T extends ThingModifiedEvent> T assertStagedModificationResult(
+            final CommandStrategy> underTest,
+            @Nullable final Thing thing,
+            final C command,
+            final Class expectedEventClass,
+            final CommandResponse expectedCommandResponse,
+            final boolean becomeDeleted) {
+
+        final CommandStrategy.Context context = getDefaultContext();
+        final Result> result = applyStrategy(underTest, context, thing, command);
+
+        return assertStagedModificationResult(result, expectedEventClass, expectedCommandResponse, becomeDeleted);
+    }
+
     protected static > void assertErrorResult(
             final CommandStrategy> underTest,
             @Nullable final Thing thing,
@@ -159,6 +175,28 @@ private static > T assertModificationResult(fina
         return event.getValue();
     }
 
+    private static > T assertStagedModificationResult(final Result> result,
+            final Class eventClazz,
+            final WithDittoHeaders expectedResponse,
+            final boolean becomeDeleted) {
+
+        final ArgumentCaptor>> eventStage = ArgumentCaptor.forClass(CompletionStage.class);
+        final ArgumentCaptor> responseStage = ArgumentCaptor.forClass(CompletionStage.class);
+
+        final ResultVisitor> mock = mock(Dummy.class);
+
+        result.accept(mock);
+
+        verify(mock).onStagedMutation(any(), eventStage.capture(), responseStage.capture(), anyBoolean(), eq(becomeDeleted));
+        assertThat(eventStage.getValue()).isInstanceOf(CompletionStage.class);
+        CompletableFutureAssert.assertThatCompletionStage(eventStage.getValue())
+                .isCompletedWithValueMatching(t -> eventClazz.isAssignableFrom(t.getClass()));
+        assertThat(responseStage.getValue()).isInstanceOf(CompletionStage.class);
+        CompletableFutureAssert.assertThatCompletionStage(responseStage.getValue())
+                .isCompletedWithValue(expectedResponse);
+        return (T) eventStage.getValue().toCompletableFuture().join();
+    }
+
     private static void assertInfoResult(final Result> result, final WithDittoHeaders infoResponse) {
         final ResultVisitor> mock = mock(Dummy.class);
         result.accept(mock);
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeatureStrategyTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeatureStrategyTest.java
index e3bc75a6bae..a549d9b8d2c 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeatureStrategyTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeatureStrategyTest.java
@@ -66,9 +66,9 @@ public void modifyFeatureOnThingWithoutFeatures() {
         final CommandStrategy.Context context = getDefaultContext();
         final ModifyFeature command = ModifyFeature.of(context.getState(), modifiedFeature, DittoHeaders.empty());
 
-        assertModificationResult(underTest, THING_V2.removeFeatures(), command,
+        assertStagedModificationResult(underTest, THING_V2.removeFeatures(), command,
                 FeatureCreated.class,
-                ETagTestUtils.modifyFeatureResponse(context.getState(), command.getFeature(), command.getDittoHeaders(), true));
+                ETagTestUtils.modifyFeatureResponse(context.getState(), command.getFeature(), command.getDittoHeaders(), true), false);
     }
 
     @Test
@@ -76,9 +76,9 @@ public void modifyFeatureOnThingWithoutThatFeature() {
         final CommandStrategy.Context context = getDefaultContext();
         final ModifyFeature command = ModifyFeature.of(context.getState(), modifiedFeature, DittoHeaders.empty());
 
-        assertModificationResult(underTest, THING_V2.removeFeature(modifiedFeature.getId()), command,
+        assertStagedModificationResult(underTest, THING_V2.removeFeature(modifiedFeature.getId()), command,
                 FeatureCreated.class,
-                ETagTestUtils.modifyFeatureResponse(context.getState(), command.getFeature(), command.getDittoHeaders(), true));
+                ETagTestUtils.modifyFeatureResponse(context.getState(), command.getFeature(), command.getDittoHeaders(), true), false);
     }
 
     @Test
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeaturesStrategyTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeaturesStrategyTest.java
index fe536d219e4..282b981dfa5 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeaturesStrategyTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ModifyFeaturesStrategyTest.java
@@ -77,9 +77,9 @@ public void modifyFeaturesOfThingWithoutFeatures() {
         final CommandStrategy.Context context = getDefaultContext();
         final ModifyFeatures command = ModifyFeatures.of(context.getState(), modifiedFeatures, DittoHeaders.empty());
 
-        assertModificationResult(underTest, THING_V2.removeFeatures(), command,
+        assertStagedModificationResult(underTest, THING_V2.removeFeatures(), command,
                 FeaturesCreated.class,
-                ETagTestUtils.modifyFeaturesResponse(context.getState(), modifiedFeatures, command.getDittoHeaders(), true));
+                ETagTestUtils.modifyFeaturesResponse(context.getState(), modifiedFeatures, command.getDittoHeaders(), true), false);
     }
 
     @Test
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingConflictStrategyTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingConflictStrategyTest.java
index 028caf5a5e8..8f909e6a59b 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingConflictStrategyTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingConflictStrategyTest.java
@@ -18,6 +18,8 @@
 import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf;
 import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable;
 
+import java.util.concurrent.CompletionStage;
+
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.base.model.headers.WithDittoHeaders;
@@ -95,8 +97,16 @@ private ExpectErrorVisitor(final Class clazz) {
         }
 
         @Override
-        public void onMutation(final Command command, final ThingEvent event, final WithDittoHeaders response,
-                final boolean becomeCreated, final boolean becomeDeleted) {
+        public void onMutation(final Command command, final ThingEvent event,
+                final WithDittoHeaders response, final boolean becomeCreated,
+                final boolean becomeDeleted) {
+            throw new AssertionError("Expect error, got mutation: " + event);
+        }
+
+        @Override
+        public void onStagedMutation(final Command command, final CompletionStage> event,
+                final CompletionStage response, final boolean becomeCreated,
+                final boolean becomeDeleted) {
             throw new AssertionError("Expect error, got mutation: " + event);
         }
 
@@ -105,6 +115,11 @@ public void onQuery(final Command command, final WithDittoHeaders response) {
             throw new AssertionError("Expect error, got query: " + response);
         }
 
+        @Override
+        public void onStagedQuery(final Command command, final CompletionStage response) {
+            throw new AssertionError("Expect error, got query: " + response);
+        }
+
         @Override
         public void onError(final DittoRuntimeException error, final Command errorCausingCommand) {
             assertThat(error).isInstanceOf(clazz);
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingsConditionalHeadersValidatorTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingsConditionalHeadersValidatorTest.java
index 1067979d098..536cc1f0c80 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingsConditionalHeadersValidatorTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/strategies/commands/ThingsConditionalHeadersValidatorTest.java
@@ -15,6 +15,7 @@
 import static java.text.MessageFormat.format;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.assertj.core.api.Assertions.assertThatNoException;
 import static org.eclipse.ditto.base.model.signals.commands.Command.Category.DELETE;
 import static org.eclipse.ditto.base.model.signals.commands.Command.Category.MODIFY;
 import static org.eclipse.ditto.base.model.signals.commands.Command.Category.QUERY;
@@ -30,15 +31,22 @@
 import org.assertj.core.api.ThrowableAssertAlternative;
 import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
+import org.eclipse.ditto.base.model.headers.IfEqual;
 import org.eclipse.ditto.base.model.headers.entitytag.EntityTag;
 import org.eclipse.ditto.base.model.headers.entitytag.EntityTagMatchers;
 import org.eclipse.ditto.base.model.signals.commands.Command;
 import org.eclipse.ditto.base.model.signals.commands.Command.Category;
 import org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator;
 import org.eclipse.ditto.json.JsonFieldSelector;
+import org.eclipse.ditto.json.JsonObject;
+import org.eclipse.ditto.json.JsonPointer;
+import org.eclipse.ditto.json.JsonValue;
+import org.eclipse.ditto.things.model.Thing;
 import org.eclipse.ditto.things.model.ThingId;
 import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingPreconditionFailedException;
 import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingPreconditionNotModifiedException;
+import org.eclipse.ditto.things.model.signals.commands.modify.MergeThing;
+import org.eclipse.ditto.things.model.signals.commands.modify.ModifyAttribute;
 import org.eclipse.ditto.things.model.signals.commands.query.RetrieveThing;
 import org.junit.Test;
 
@@ -159,6 +167,75 @@ public void assertThrowingNotModifiedWhenSelectedFieldDoesNotContainPolicy() {
         assertion.satisfies(exception -> assertETagHeaderInDre(exception, actualEntityTag));
     }
 
+    @Test
+    public void ifEqualDoesThrowExceptionWhenIfEqualSkipAndValueIsEqual() {
+        final ThingId thingId = ThingId.generateRandom();
+        final JsonPointer attributePath = JsonPointer.of("foo/bar");
+        final JsonValue attributeValue = JsonValue.of(false);
+        final Thing thing = Thing.newBuilder()
+                .setId(thingId)
+                .setAttribute(attributePath, attributeValue)
+                .build();
+        final ModifyAttribute command = ModifyAttribute.of(thingId, attributePath, attributeValue,
+                DittoHeaders.newBuilder().ifEqual(IfEqual.SKIP).build());
+
+        assertThatExceptionOfType(ThingPreconditionNotModifiedException.class)
+                .isThrownBy(() -> SUT.applyIfEqualHeader(command, thing))
+                .withMessage("The previous value was equal to the new value and the 'if-equal' header was set to 'skip'.");
+    }
+
+    @Test
+    public void ifEqualDoesThrowExceptionWhenIfEqualSkipAndValueIsEqualUsingMerge() {
+        final ThingId thingId = ThingId.generateRandom();
+        final JsonPointer attributePath = JsonPointer.of("foo/bar");
+        final JsonValue attributeValue = JsonValue.of(false);
+        final Thing thing = Thing.newBuilder()
+                .setId(thingId)
+                .setAttribute(attributePath, attributeValue)
+                .build();
+        final MergeThing command = MergeThing.of(thingId, JsonPointer.empty(), thing.toJson(),
+                DittoHeaders.newBuilder().ifEqual(IfEqual.SKIP).build());
+
+        assertThatExceptionOfType(ThingPreconditionNotModifiedException.class)
+                .isThrownBy(() -> SUT.applyIfEqualHeader(command, thing))
+                .withMessage("The previous value was equal to the new value and the 'if-equal' header was set to 'skip'.");
+    }
+
+    @Test
+    public void ifEqualDoesNotThrowExceptionWhenIfEqualSkipAndValueIsNotEqual() {
+        final ThingId thingId = ThingId.generateRandom();
+        final JsonPointer attributePath = JsonPointer.of("foo/bar");
+        final JsonValue attributeValue = JsonValue.of(false);
+        final Thing thing = Thing.newBuilder()
+                .setId(thingId)
+                .setAttribute(attributePath, attributeValue)
+                .build();
+        final ModifyAttribute command = ModifyAttribute.of(thingId, attributePath, JsonValue.of(true),
+                DittoHeaders.newBuilder().ifEqual(IfEqual.SKIP).build());
+
+        assertThatNoException()
+                .isThrownBy(() -> SUT.applyIfEqualHeader(command, thing));
+    }
+
+    @Test
+    public void ifEqualDoesNotThrowExceptionWhenIfEqualUpdateAndValueIsEqual() {
+        final ThingId thingId = ThingId.generateRandom();
+        final JsonPointer attributePath = JsonPointer.of("foo/bar");
+        final JsonValue attributeValue = JsonObject.newBuilder()
+                .set("foo", false)
+                .set("bar", "yeesss")
+                .build();
+        final Thing thing = Thing.newBuilder()
+                .setId(thingId)
+                .setAttribute(attributePath, attributeValue)
+                .build();
+        final ModifyAttribute command = ModifyAttribute.of(thingId, attributePath, attributeValue,
+                DittoHeaders.newBuilder().ifEqual(IfEqual.UPDATE).build());
+
+        assertThatNoException()
+                .isThrownBy(() -> SUT.applyIfEqualHeader(command, thing));
+    }
+
     private RetrieveThing createRetrieveThingCommand(final String ifNoneMatchHeaderValue, final String selectedFields) {
 
         final JsonFieldSelector fieldSelector = JsonFieldSelector.newInstance(selectedFields);
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapterTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapterTest.java
index a6b097f5de1..062f530f58f 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapterTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapterTest.java
@@ -17,6 +17,7 @@
 
 import java.time.Instant;
 import java.util.List;
+import java.util.Set;
 
 import org.bson.BsonDocument;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
@@ -30,7 +31,12 @@
 import org.eclipse.ditto.things.model.signals.events.ThingDeleted;
 import org.junit.Test;
 
+import com.typesafe.config.ConfigFactory;
+
+import akka.actor.ActorSystem;
+import akka.actor.ExtendedActorSystem;
 import akka.persistence.journal.EventSeq;
+import akka.persistence.journal.Tagged;
 import scala.jdk.javaapi.CollectionConverters;
 
 /**
@@ -41,7 +47,8 @@ public final class ThingMongoEventAdapterTest {
     private final ThingMongoEventAdapter underTest;
 
     public ThingMongoEventAdapterTest() {
-        underTest = new ThingMongoEventAdapter(null);
+        underTest = new ThingMongoEventAdapter(
+                (ExtendedActorSystem) ActorSystem.create("test", ConfigFactory.load("test")));
     }
 
     @Test
@@ -93,11 +100,13 @@ public void toJournalReturnsBsonDocument() {
                 "                    \"attributes\" : {\n" +
                 "                        \"hello\" : \"cloud\"\n" +
                 "                    }\n" +
-                "                }\n" +
+                "                },\n" +
+                "                \"__hh\": {}\n" +
                 "            }";
         final BsonDocument bsonEvent = BsonDocument.parse(journalEntry);
+        final Tagged tagged = new Tagged(bsonEvent, Set.of());
 
-        assertThat(underTest.toJournal(thingCreated)).isEqualTo(bsonEvent);
+        assertThat(underTest.toJournal(thingCreated)).isEqualTo(tagged);
     }
 
     @Test
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalCommandRegistryTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalCommandRegistryTest.java
index 8aea6bf6ce3..0941014ce8a 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalCommandRegistryTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalCommandRegistryTest.java
@@ -17,6 +17,7 @@
 import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand;
 import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence;
 import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace;
+import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents;
 import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver;
 import org.eclipse.ditto.internal.models.streaming.SudoStreamPids;
 import org.eclipse.ditto.internal.utils.health.RetrieveHealth;
@@ -55,6 +56,8 @@ public ThingsServiceGlobalCommandRegistryTest() {
                 PurgeEntities.class,
                 ModifySplitBrainResolver.class,
                 PublishSignal.class,
+                SubscribeForPersistedEvents.class,
+                PublishSignal.class,
                 CreateSubscription.class,
                 QueryThings.class,
                 SudoCountThings.class
diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalEventRegistryTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalEventRegistryTest.java
index f466d1b1c06..91c529610b0 100644
--- a/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalEventRegistryTest.java
+++ b/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalEventRegistryTest.java
@@ -12,10 +12,11 @@
  */
 package org.eclipse.ditto.things.service.starter;
 
-import org.eclipse.ditto.policies.model.signals.events.PolicyModified;
-import org.eclipse.ditto.things.api.ThingSnapshotTaken;
+import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete;
 import org.eclipse.ditto.internal.utils.persistentactors.EmptyEvent;
 import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases;
+import org.eclipse.ditto.policies.model.signals.events.PolicyModified;
+import org.eclipse.ditto.things.api.ThingSnapshotTaken;
 import org.eclipse.ditto.things.model.signals.events.FeatureDeleted;
 import org.eclipse.ditto.thingsearch.api.events.ThingsOutOfSync;
 import org.eclipse.ditto.thingsearch.model.signals.events.SubscriptionCreated;
@@ -28,6 +29,8 @@ public ThingsServiceGlobalEventRegistryTest() {
                 ThingSnapshotTaken.class,
                 EmptyEvent.class,
                 PolicyModified.class,
+                StreamingSubscriptionComplete.class,
+                PolicyModified.class,
                 SubscriptionCreated.class,
                 ThingsOutOfSync.class
         );
diff --git a/things/service/src/test/resources/test.conf b/things/service/src/test/resources/test.conf
index 48e080c843b..db4f32dd8a9 100755
--- a/things/service/src/test/resources/test.conf
+++ b/things/service/src/test/resources/test.conf
@@ -170,6 +170,10 @@ wot-dispatcher {
   type = Dispatcher
   executor = "thread-pool-executor"
 }
+wot-dispatcher-cache-loader {
+  type = Dispatcher
+  executor = "thread-pool-executor"
+}
 
 blocked-namespaces-dispatcher {
   type = Dispatcher
diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchProperty.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchProperty.java
index 166c9ceb140..123394aed9f 100755
--- a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchProperty.java
+++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchProperty.java
@@ -240,6 +240,13 @@ public PropertySearchFilter like(final String value) {
         return ImmutablePropertyFilter.of(SearchFilter.Type.LIKE, propertyPath, JsonFactory.newValue(value));
     }
 
+    @Override
+    public PropertySearchFilter ilike(final String value) {
+        checkStringValue(value);
+
+        return ImmutablePropertyFilter.of(SearchFilter.Type.ILIKE, propertyPath, JsonFactory.newValue(value));
+    }
+
     @Override
     public PropertySearchFilter in(final boolean value, final Boolean... furtherValues) {
         return in(toCollection(JsonFactory::newValue, value, furtherValues));
diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchFilter.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchFilter.java
index 57d47a3c347..d56e192ca9c 100755
--- a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchFilter.java
+++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchFilter.java
@@ -99,6 +99,11 @@ enum Type {
          */
         LIKE("like"),
 
+        /**
+         * Filter type for checking if a string matches a regular expression, in a case insensitive way.
+         */
+        ILIKE("ilike"),
+
         /**
          * Filter type for checking if an entity is contained in a set of given entities.
          */
diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchProperty.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchProperty.java
index 4a2850868c2..5f9dccd95f3 100755
--- a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchProperty.java
+++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchProperty.java
@@ -293,6 +293,15 @@ public interface SearchProperty {
      */
     PropertySearchFilter like(String value);
 
+    /**
+     * Returns a new search filter for checking if the value of this property is case insensitive like the given value.
+     *
+     * @param value the value to compare the value of this property with.
+     * @return the new search filter.
+     * @throws NullPointerException if {@code value} is {@code null}.
+     */
+    PropertySearchFilter ilike(String value);
+
     /**
      * Returns a new search filter for checking if the value of this property is in the given value(s).
      *
diff --git a/thingsearch/model/src/test/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchPropertyTest.java b/thingsearch/model/src/test/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchPropertyTest.java
index 3e1b07c90dc..0ce96aa648e 100755
--- a/thingsearch/model/src/test/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchPropertyTest.java
+++ b/thingsearch/model/src/test/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchPropertyTest.java
@@ -207,6 +207,14 @@ public void likeReturnsExpected() {
                 .hasStringRepresentation("like(" + PROPERTY_PATH + ",\"" + BOSCH + "\")");
     }
 
+    @Test
+    public void ilikeReturnsExpected() {
+        assertThat(underTest.ilike(BOSCH))
+                .hasType(SearchFilter.Type.ILIKE)
+                .hasOnlyValue(BOSCH)
+                .hasStringRepresentation("ilike(" + PROPERTY_PATH + ",\"" + BOSCH + "\")");
+    }
+
     @Test(expected = NullPointerException.class)
     public void tryToCallInWithNullStringForMandatoryValue() {
         underTest.in(null, ACME);
diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java
index 22bf085b85a..09154ae9e54 100644
--- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java
+++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java
@@ -19,6 +19,7 @@
 import java.util.List;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.regex.Pattern;
 
 import javax.annotation.Nullable;
 
@@ -137,6 +138,18 @@ public Function visitLike(final String value) {
         return fieldName -> Filters.regex(fieldName, valueWithoutLeadingOrTrailingWildcard, "");
     }
 
+    @Override
+    public Function visitILike(final String value) {
+        // remove leading or trailing wildcard because queries like /^a/ are much faster than /^a.*$/ or /^a.*/
+        // from mongodb docs:
+        // "Additionally, while /^a/, /^a.*/, and /^a.*$/ match equivalent strings, they have different performance
+        // characteristics. All of these expressions use an index if an appropriate index exists;
+        // however, /^a.*/, and /^a.*$/ are slower. /^a/ can stop scanning after matching the prefix."
+        final String valueWithoutLeadingOrTrailingWildcard = removeLeadingOrTrailingWildcard(value);
+        Pattern pattern = Pattern.compile(valueWithoutLeadingOrTrailingWildcard, Pattern.CASE_INSENSITIVE);
+        return fieldName -> Filters.regex(fieldName, pattern);
+    }
+
     private static String removeLeadingOrTrailingWildcard(final String valueString) {
         String valueWithoutLeadingOrTrailingWildcard = valueString;
         if (valueString.startsWith(LEADING_WILDCARD)) {
diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/write/streaming/BackgroundSyncStream.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/write/streaming/BackgroundSyncStream.java
index 24408f24ad3..4842c267655 100644
--- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/write/streaming/BackgroundSyncStream.java
+++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/write/streaming/BackgroundSyncStream.java
@@ -36,6 +36,7 @@
 import org.eclipse.ditto.policies.model.PolicyConstants;
 import org.eclipse.ditto.policies.model.PolicyId;
 import org.eclipse.ditto.policies.model.PolicyImport;
+import org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyNotAccessibleException;
 import org.eclipse.ditto.things.model.ThingConstants;
 import org.eclipse.ditto.things.model.ThingId;
 import org.eclipse.ditto.thingsearch.service.persistence.write.model.Metadata;
@@ -53,7 +54,7 @@
  */
 public final class BackgroundSyncStream {
 
-    private static Logger LOGGER = DittoLoggerFactory.getThreadSafeLogger(BackgroundSyncStream.class);
+    private static final Logger LOGGER = DittoLoggerFactory.getThreadSafeLogger(BackgroundSyncStream.class);
     private static final ThingId EMPTY_THING_ID = ThingId.of(LowerBound.emptyEntityId(ThingConstants.ENTITY_TYPE));
     private static final PolicyId EMPTY_POLICY_ID = PolicyId.of(LowerBound.emptyEntityId(PolicyConstants.ENTITY_TYPE));
 
@@ -186,7 +187,7 @@ private Source emitUnlessConsistent(final Metadata persisted,
                         .orElseGet(() -> CompletableFuture.completedStage(true));
                 return Source.completionStage(consistencyCheckCs)
                         .flatMapConcat(policiesAreConsistent -> {
-                            if (policiesAreConsistent) {
+                            if (Boolean.TRUE.equals(policiesAreConsistent)) {
                                 return Source.empty();
                             } else {
                                 return Source.single(indexed.invalidateCaches(false, true)).log("PoliciesInconsistent");
@@ -283,13 +284,17 @@ private CompletionStage isPolicyTagUpToDate(final PolicyTag policyTag)
                         final String message = String.format(
                                 "Error in background sync stream when trying to retrieve policy revision of " +
                                         "Policy with ID <%s>", policyId);
-                        LOGGER.error(message, error);
+                        LOGGER.warn(message, error);
                         return false;
                     } else if (response instanceof SudoRetrievePolicyRevisionResponse retrieveResponse) {
                         final long revision = retrieveResponse.getRevision();
                         return policyTag.getRevision() == revision;
+                    } else if (response instanceof PolicyNotAccessibleException) {
+                        LOGGER.info("Policy with ID <{}> is/was no longer accessible when trying to retrieve policy " +
+                                "revision of Policy", policyId);
+                        return false;
                     } else {
-                        LOGGER.error("Unexpected message in background sync stream when trying to retrieve policy " +
+                        LOGGER.warn("Unexpected message in background sync stream when trying to retrieve policy " +
                                         "revision of Policy with ID <{}>. Expected <{}> but got <{}>.",
                                 policyId, SudoRetrievePolicyRevisionResponse.class, response.getClass());
                         return false;
@@ -305,12 +310,16 @@ private CompletionStage> retrievePolicy(final PolicyId policyId
                     if (error != null) {
                         final String message = String.format("Error in background sync stream when trying to " +
                                 "retrieve policy with ID <%s>", policyId);
-                        LOGGER.error(message, error);
+                        LOGGER.warn(message, error);
                         return Optional.empty();
                     } else if (response instanceof SudoRetrievePolicyResponse retrieveResponse) {
                         return Optional.of(retrieveResponse.getPolicy());
+                    } else if (response instanceof PolicyNotAccessibleException) {
+                        LOGGER.info("Policy with ID <{}> is/was no longer accessible when trying to retrieve policy",
+                                policyId);
+                        return Optional.empty();
                     } else {
-                        LOGGER.error("Unexpected message in background sync stream when trying to retrieve policy " +
+                        LOGGER.warn("Unexpected message in background sync stream when trying to retrieve policy " +
                                         "with ID <{}>. Expected <{}> but got <{}>.",
                                 policyId, SudoRetrievePolicyRevisionResponse.class, response.getClass());
                         return Optional.empty();
diff --git a/thingsearch/service/src/main/resources/search.conf b/thingsearch/service/src/main/resources/search.conf
index fcc7ff75317..1dcec5529ea 100755
--- a/thingsearch/service/src/main/resources/search.conf
+++ b/thingsearch/service/src/main/resources/search.conf
@@ -113,22 +113,28 @@ ditto {
       force-update-after-start-random-factor = ${?FORCE_UPDATE_AFTER_START_RANDOM_FACTOR}
 
       background-sync {
+        # whether background sync is turned on
         enabled = true
         enabled = ${?BACKGROUND_SYNC_ENABLED}
 
+        # duration between service start-up and the beginning of background sync
         quiet-period = 8m
         quiet-period = ${?BACKGROUND_SYNC_QUIET_PERIOD}
 
+        # how soon to close the remote stream if no element passed through it
         idle-timeout = 5m
         idle-timeout = ${?BACKGROUND_SYNC_IDLE_TIMEOUT}
 
+        # how long to wait before reacting to out-of-date search index entries
         tolerance-window = 20m
         tolerance-window = ${?BACKGROUND_SYNC_TOLERANCE_WINDOW}
 
+        # how long to wait for the policy shard region for the most up-to-date policy revision
         policy-ask-timeout = 10s
         policy-ask-timeout = ${?BACKGROUND_SYNC_POLICY_ASK_TIMEOUT}
 
         keep {
+          # how many events to keep in the actor state
           events = 50
           events = ${?BACKGROUND_SYNC_KEEP_EVENTS}
         }
@@ -169,7 +175,7 @@ ditto {
 
         # delay before updater actor is stopped after receiving thing deleted event
         thing-deletion-timeout = 5m
-        thing-deletion-timeout = ${?THINGS_SEARCH_UPDATER_STREAM_WRITE_INTERVAL}
+        thing-deletion-timeout = ${?THINGS_SEARCH_UPDATER_STREAM_THING_DELETION_TIMEOUT}
 
         # configuration for retrieval of policies/things via sharding
         ask-with-retry {
diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java
index ff8cdc81f97..512235581bc 100644
--- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java
+++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java
@@ -17,6 +17,7 @@
 import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand;
 import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence;
 import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace;
+import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents;
 import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver;
 import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection;
 import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection;
@@ -60,7 +61,8 @@ public ThingsSearchServiceGlobalCommandRegistryTest() {
                 CleanupPersistence.class,
                 ModifyConnection.class,
                 ModifySplitBrainResolver.class,
-                RetrieveConnection.class
+                RetrieveConnection.class,
+                SubscribeForPersistedEvents.class
         );
     }
 
diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalEventRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalEventRegistryTest.java
index 1c89950ff16..b0913cf70dc 100644
--- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalEventRegistryTest.java
+++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalEventRegistryTest.java
@@ -12,6 +12,7 @@
  */
 package org.eclipse.ditto.thingsearch.service.starter;
 
+import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete;
 import org.eclipse.ditto.connectivity.model.signals.events.ConnectionModified;
 import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases;
 import org.eclipse.ditto.policies.model.signals.events.ResourceDeleted;
@@ -29,7 +30,8 @@ public ThingsSearchServiceGlobalEventRegistryTest() {
                 SubscriptionCreated.class,
                 ThingsOutOfSync.class,
                 ThingSnapshotTaken.class,
-                ConnectionModified.class
+                ConnectionModified.class,
+                StreamingSubscriptionComplete.class
         );
     }
 
diff --git a/ui/Dockerfile b/ui/Dockerfile
index 608321e6650..aae579baa67 100644
--- a/ui/Dockerfile
+++ b/ui/Dockerfile
@@ -14,7 +14,4 @@ FROM nginxinc/nginx-unprivileged:alpine
 WORKDIR /usr/share/nginx/html
 
 COPY ./index.html           .
-COPY ./index.css            .
-COPY ./main.js              .
-COPY ./modules              ./modules
-COPY ./templates            ./templates
+COPY ./dist                 ./dist
diff --git a/ui/build.mjs b/ui/build.mjs
new file mode 100644
index 00000000000..dd12376c81d
--- /dev/null
+++ b/ui/build.mjs
@@ -0,0 +1,29 @@
+import {argv} from 'node:process';
+import * as esbuild from 'esbuild';
+import {sassPlugin} from 'esbuild-sass-plugin';
+
+const config = {
+  entryPoints: ['main.js'],
+  bundle: true,
+  outdir: 'dist',
+  loader: {
+    '.html': 'text',
+  },
+  plugins: [sassPlugin()],
+};
+
+if (argv[2] === 'serve') {
+  config.sourcemap = true;
+
+  const ctx = await esbuild.context(config);
+
+  let {host, port} = await ctx.serve({
+    servedir: '.',
+  });
+} else {
+  config.minify = true;
+
+  await esbuild.build(config);
+}
+
+
diff --git a/ui/index.html b/ui/index.html
index 4778015816b..813efd48036 100644
--- a/ui/index.html
+++ b/ui/index.html
@@ -16,13 +16,10 @@
     Eclipse Ditto™ explorer
     
     
-    
     
-    
     
@@ -32,14 +29,11 @@
     
-    
-    
-    
-    
+    
+    
+    
 
 
 
@@ -64,11 +58,16 @@
                             Policies
                         
                     
-                    
+                    
                     
                         
  • 
-                            CRUD Thing
+                            
+                                CRUD Thing!
                         
    • 
                     
                     
    
                         
    
                             
    
                                 
-                                    
+                                    
    
                                 
    
                             
    
                     
    
                     
    
-                        
    
-                            
    
-                                
-                                
-                                
-                                
-                                
-                                
    
-                            
    
+                        
                             
    
-                                
+                                
                                 
    
                                     
+                                    data-bs-toggle="dropdown" disabled id="buttonThingDefinitions">
                                                                         
                                 
    
-                                
+                                
                             
    
-                            
    
+                            
    
                                 
    
                             
    
-                        
    
+                            
                     
    
                 
    
             
    
         
     
 
-
    Attributes 
    
+
    Attributes 
    
 
    
 
    
     
    
         
    
             
-                
+                
    
             
    
         
    
         
    
             
             
    
                 
    
-                    
    
-                        
-                        
-                        
-                        
-                    
    
-                    
    
-                        
-                        
-                    
    
+                    
+                        
    
+                            
+                            
+                            
    
+                        
    
+                        
                 
    
             
    
         
    
diff --git a/ui/modules/things/things.js b/ui/modules/things/things.js
index da397b1a179..0e58b670537 100644
--- a/ui/modules/things/things.js
+++ b/ui/modules/things/things.js
@@ -1,53 +1,36 @@
-/* eslint-disable require-jsdoc */
-/*
- * Copyright (c) 2022 Contributors to the Eclipse Foundation
- *
- * See the NOTICE file(s) distributed with this work for additional
- * information regarding copyright ownership.
- *
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Public License 2.0 which is available at
- * http://www.eclipse.org/legal/epl-2.0
- *
- * SPDX-License-Identifier: EPL-2.0
- */
-
 /* eslint-disable new-cap */
-/* eslint-disable no-invalid-this */
-import {JSONPath} from 'https://cdn.jsdelivr.net/npm/jsonpath-plus@5.0.3/dist/index-browser-esm.min.js';
+/*
+* Copyright (c) 2022 Contributors to the Eclipse Foundation
+*
+* See the NOTICE file(s) distributed with this work for additional
+* information regarding copyright ownership.
+*
+* This program and the accompanying materials are made available under the
+* terms of the Eclipse Public License 2.0 which is available at
+* http://www.eclipse.org/legal/epl-2.0
+*
+* SPDX-License-Identifier: EPL-2.0
+*/
+/* eslint-disable require-jsdoc */
+// @ts-check
 import * as API from '../api.js';
 
-import * as Environments from '../environments/environments.js';
-import * as SearchFilter from './searchFilter.js';
 import * as Utils from '../utils.js';
-import * as Fields from './fields.js';
+import {TabHandler} from '../utils/tabHandler.js';
+import * as ThingsSearch from './thingsSearch.js';
+import thingsHTML from './things.html';
 
 export let theThing;
-let theSearchCursor;
-
-let thingJsonEditor;
-
-let thingTemplates;
 
 const observers = [];
 
 const dom = {
-  thingsTableHead: null,
-  thingsTableBody: null,
-  thingDetails: null,
-  thingId: null,
-  buttonCreateThing: null,
-  buttonSaveThing: null,
-  buttonDeleteThing: null,
-  inputThingDefinition: null,
-  ulThingDefinitions: null,
-  tabModifyThing: null,
-  searchFilterEdit: null,
   collapseThings: null,
   tabThings: null,
 };
 
-const uuidRegex = /([0-9a-f]{7})[0-9a-f]-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;
+document.getElementById('thingsHTML').innerHTML = thingsHTML;
+
 
 /**
  * Adds a listener function for the currently selected thing
@@ -61,220 +44,8 @@ export function addChangeListener(observer) {
  * Initializes components. Should be called after DOMContentLoaded event
  */
 export async function ready() {
-  Environments.addChangeListener(onEnvironmentChanged);
-
   Utils.getAllElementsById(dom);
-
-  thingJsonEditor = Utils.createAceEditor('thingJsonEditor', 'ace/mode/json');
-
-  loadThingTemplates();
-
-  dom.ulThingDefinitions.addEventListener('click', (event) => {
-    setTheThing(null);
-    Utils.tableAdjustSelection(dom.thingsTableBody, () => false);
-    dom.inputThingDefinition.value = event.target.textContent;
-    thingJsonEditor.setValue(JSON.stringify(thingTemplates[event.target.textContent], null, 2), -1);
-  });
-
-  dom.searchFilterEdit.onchange = removeMoreFromThingList;
-
-  dom.buttonCreateThing.onclick = async () => {
-    const editorValue = thingJsonEditor.getValue();
-    if (dom.thingId.value !== undefined && dom.thingId.value !== '') {
-      API.callDittoREST('PUT',
-          '/things/' + dom.thingId.value,
-          editorValue === '' ? {} : JSON.parse(editorValue),
-          {
-            'if-none-match': '*',
-          },
-      ).then((data) => {
-        refreshThing(data.thingId, () => {
-          getThings([data.thingId]);
-        });
-      });
-    } else {
-      API.callDittoREST('POST', '/things', editorValue === '' ? {} : JSON.parse(editorValue))
-          .then((data) => {
-            refreshThing(data.thingId, () => {
-              getThings([data.thingId]);
-            });
-          });
-    }
-  };
-
-  dom.buttonSaveThing.onclick = () => {
-    Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId);
-    modifyThing('PUT');
-  };
-
-  dom.buttonDeleteThing.onclick = () => {
-    Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId);
-    Utils.confirm(`Are you sure you want to delete thing
    '${theThing.thingId}'?`, 'Delete', () => {
-      modifyThing('DELETE');
-    });
-  };
-
-  dom.thingsTableBody.addEventListener('click', (event) => {
-    if (event.target && event.target.nodeName === 'TD') {
-      const row = event.target.parentNode;
-      if (row.id === 'searchThingsMore') {
-        row.style.pointerEvents = 'none';
-        event.stopImmediatePropagation();
-        searchThings(dom.searchFilterEdit.value, theSearchCursor);
-      } else {
-        if (theThing && theThing.thingId === row.id) {
-          setTheThing(null);
-        } else {
-          refreshThing(row.id);
-        }
-      }
-    }
-  });
-
-  document.querySelector('a[data-bs-target="#tabModifyThing"]').addEventListener('shown.bs.tab', (event) => {
-    thingJsonEditor.renderer.updateFull();
-  });
-
-  dom.tabThings.onclick = onTabActivated;
-
-  dom.searchFilterEdit.focus();
-}
-
-function loadThingTemplates() {
-  fetch('templates/thingTemplates.json')
-      .then((response) => {
-        response.json().then((loadedTemplates) => {
-          thingTemplates = loadedTemplates;
-          Utils.addDropDownEntries(dom.ulThingDefinitions, Object.keys(thingTemplates));
-        });
-      });
-}
-
-/**
- * Fills the things table UI with the given things
- * @param {Array} thingsList Array of thing json objects
- */
-function fillThingsTable(thingsList) {
-  const activeFields = Environments.current().fieldList.filter((f) => f.active);
-  fillHeaderRow();
-  let thingSelected = false;
-  thingsList.forEach((item, t) => {
-    const row = dom.thingsTableBody.insertRow();
-    fillBodyRow(row, item);
-  });
-  if (!thingSelected) {
-    setTheThing(null);
-  }
-
-  function fillHeaderRow() {
-    dom.thingsTableHead.innerHTML = '';
-    // Utils.addCheckboxToRow(dom.thingsTableHead, 'checkboxHead', false, null);
-    Utils.insertHeaderCell(dom.thingsTableHead, '');
-    Utils.insertHeaderCell(dom.thingsTableHead, 'Thing ID');
-    activeFields.forEach((field) => {
-      Utils.insertHeaderCell(dom.thingsTableHead, field['label'] ? field.label : field.path);
-    });
-  }
-
-  function fillBodyRow(row, item) {
-    row.id = item.thingId;
-    if (theThing && (item.thingId === theThing.thingId)) {
-      thingSelected = true;
-      row.classList.add('table-active');
-    }
-    Utils.addCheckboxToRow(
-        row,
-        item.thingId,
-        Environments.current().pinnedThings.includes(item.thingId),
-        togglePinnedThing,
-    );
-    Utils.addCellToRow(row, beautifyId(item.thingId), item.thingId);
-    activeFields.forEach((field) => {
-      let path = field.path.replace(/\//g, '.');
-      if (path.charAt(0) !== '.') {
-        path = '$.' + path;
-      }
-      const elem = JSONPath({
-        json: item,
-        path: path,
-      });
-      Utils.addCellToRow(row, elem.length !== 0 ? elem[0] : '');
-    });
-  }
-
-  function beautifyId(longId) {
-    let result = longId;
-    if (Environments.current()['shortenUUID']) {
-      result = result.replace(uuidRegex, '$1');
-    }
-    if (Environments.current()['defaultNamespace']) {
-      result = result.replace(Environments.current()['defaultNamespace'], 'dn');
-    }
-    return result;
-  }
-}
-
-/**
- * Calls Ditto search api and fills UI with the result
- * @param {String} filter Ditto search filter (rql)
- * @param {String} cursor (optional) cursor returned from things search for additional pages
- */
-export function searchThings(filter, cursor) {
-  document.body.style.cursor = 'progress';
-
-  const namespaces = Environments.current().searchNamespaces;
-
-  API.callDittoREST('GET',
-      '/search/things?' + Fields.getQueryParameter() +
-      ((filter && filter !== '') ? '&filter=' + encodeURIComponent(filter) : '') +
-      ((namespaces && namespaces !== '') ? '&namespaces=' + namespaces : '') +
-      '&option=sort(%2BthingId)' +
-      // ',size(3)' +
-      (cursor ? ',cursor(' + cursor + ')' : ''),
-  ).then((searchResult) => {
-    if (cursor) {
-      removeMoreFromThingList();
-    } else {
-      theSearchCursor = null;
-      dom.thingsTableBody.innerHTML = '';
-    }
-    fillThingsTable(searchResult.items);
-    checkMorePages(searchResult);
-  }).catch((error) => {
-    theSearchCursor = null;
-    dom.thingsTableBody.innerHTML = '';
-  }).finally(() => {
-    document.body.style.cursor = 'default';
-  });
-}
-
-/**
- * Gets things from Ditto by thingIds and fills the UI with the result
- * @param {Array} thingIds Array of thingIds
- */
-export function getThings(thingIds) {
-  dom.thingsTableBody.innerHTML = '';
-  if (thingIds.length > 0) {
-    API.callDittoREST('GET',
-        `/things?${Fields.getQueryParameter()}&ids=${thingIds}&option=sort(%2BthingId)`,
-    ).then(fillThingsTable);
-  }
-}
-
-/**
- * Returns a click handler for Update thing and delete thing
- * @param {String} method PUT or DELETE
- */
-function modifyThing(method) {
-  API.callDittoREST(method,
-      '/things/' + dom.thingId.value,
-      method === 'PUT' ? JSON.parse(thingJsonEditor.getValue()) : null,
-      {
-        'if-match': '*',
-      },
-  ).then(() => {
-    method === 'PUT' ? refreshThing(dom.thingId.value) : SearchFilter.performLastSearch();
-  });
+  TabHandler(dom.tabThings, dom.collapseThings, refreshView);
 }
 
 /**
@@ -283,6 +54,7 @@ function modifyThing(method) {
  * @param {function} successCallback callback function that is called after refresh is finished
  */
 export function refreshThing(thingId, successCallback) {
+  console.assert(thingId && thingId !== '', 'thingId expected');
   API.callDittoREST('GET',
       `/things/${thingId}?` +
       'fields=thingId%2CpolicyId%2Cdefinition%2Cattributes%2Cfeatures%2C_created%2C_modified%2C_revision')
@@ -297,114 +69,13 @@ export function refreshThing(thingId, successCallback) {
  * Update all UI components for the given Thing
  * @param {Object} thingJson Thing json
  */
-function setTheThing(thingJson) {
+export function setTheThing(thingJson) {
+  const isNewThingId = thingJson && (!theThing || theThing.thingId !== thingJson.thingId);
   theThing = thingJson;
-
-  updateThingDetailsTable();
-  updateThingJsonEditor();
-
-  observers.forEach((observer) => observer.call(null, theThing));
-
-  function updateThingDetailsTable() {
-    dom.thingDetails.innerHTML = '';
-    if (theThing) {
-      Utils.addTableRow(dom.thingDetails, 'thingId', false, true, theThing.thingId);
-      Utils.addTableRow(dom.thingDetails, 'policyId', false, true, theThing.policyId);
-      Utils.addTableRow(dom.thingDetails, 'definition', false, true, theThing.definition ?? '');
-      Utils.addTableRow(dom.thingDetails, 'revision', false, true, theThing._revision);
-      Utils.addTableRow(dom.thingDetails, 'created', false, true, theThing._created);
-      Utils.addTableRow(dom.thingDetails, 'modified', false, true, theThing._modified);
-    }
-  }
-
-  function updateThingJsonEditor() {
-    if (theThing) {
-      dom.thingId.value = theThing.thingId;
-      dom.inputThingDefinition.value = theThing.definition ?? '';
-      const thingCopy = JSON.parse(JSON.stringify(theThing));
-      delete thingCopy['_revision'];
-      delete thingCopy['_created'];
-      delete thingCopy['_modified'];
-      thingJsonEditor.setValue(JSON.stringify(thingCopy, null, 2), -1);
-    } else {
-      dom.thingId.value = null;
-      dom.inputThingDefinition.value = null;
-      thingJsonEditor.setValue('');
-    }
-  }
-}
-
-/**
- * Updates UI depepending on existing additional pages on Ditto things search
- * @param {Object} searchResult Result from Ditto thing search
- */
-function checkMorePages(searchResult) {
-  if (searchResult['cursor']) {
-    addMoreToThingList();
-    theSearchCursor = searchResult.cursor;
-  } else {
-    theSearchCursor = null;
-  }
-}
-
-/**
- * Adds a clickable "more" line to the things table UI
- */
-function addMoreToThingList() {
-  const moreCell = dom.thingsTableBody.insertRow().insertCell(-1);
-  moreCell.innerHTML = 'load more...';
-  moreCell.colSpan = dom.thingsTableBody.rows[0].childElementCount;
-  moreCell.style.textAlign = 'center';
-  moreCell.style.cursor = 'pointer';
-  moreCell.disabled = true;
-  moreCell.style.color = '#3a8c9a';
-  moreCell.parentNode.id = 'searchThingsMore';
-}
-
-/**
- * remove the "more" line from the things table
- */
-function removeMoreFromThingList() {
-  const moreRow = document.getElementById('searchThingsMore');
-  if (moreRow) {
-    moreRow.parentNode.removeChild(moreRow);
-  }
-}
-
-function togglePinnedThing(evt) {
-  if (evt.target.checked) {
-    Environments.current().pinnedThings.push(this.id);
-  } else {
-    const index = Environments.current().pinnedThings.indexOf(this.id);
-    if (index > -1) {
-      Environments.current().pinnedThings.splice(index, 1);
-    }
-  }
-  Environments.environmentsJsonChanged('pinnedThings');
-}
-
-let viewDirty = false;
-
-function onTabActivated() {
-  if (viewDirty) {
-    refreshView();
-    viewDirty = false;
-  }
-  // dom.searchFilterEdit.focus();
-}
-
-function onEnvironmentChanged(modifiedField) {
-  if (!['pinnedThings', 'filterList', 'messageTemplates'].includes(modifiedField)) {
-    if (dom.collapseThings.classList.contains('show')) {
-      refreshView();
-    } else {
-      viewDirty = true;
-    }
-  }
+  observers.forEach((observer) => observer.call(null, theThing, isNewThingId));
 }
 
 function refreshView() {
-  SearchFilter.performLastSearch();
+  ThingsSearch.performLastSearch();
 }
 
-
diff --git a/ui/modules/things/thingsCRUD.js b/ui/modules/things/thingsCRUD.js
new file mode 100644
index 00000000000..3ea9e4f0e42
--- /dev/null
+++ b/ui/modules/things/thingsCRUD.js
@@ -0,0 +1,195 @@
+/*
+* Copyright (c) 2022 Contributors to the Eclipse Foundation
+*
+* See the NOTICE file(s) distributed with this work for additional
+* information regarding copyright ownership.
+*
+* This program and the accompanying materials are made available under the
+* terms of the Eclipse Public License 2.0 which is available at
+* http://www.eclipse.org/legal/epl-2.0
+*
+* SPDX-License-Identifier: EPL-2.0
+*/
+/* eslint-disable require-jsdoc */
+// @ts-check
+import * as API from '../api.js';
+
+import * as Utils from '../utils.js';
+import * as ThingsSearch from './thingsSearch.js';
+import * as Things from './things.js';
+import thingTemplates from './thingTemplates.json';
+
+let thingJsonEditor;
+
+let eTag;
+
+const dom = {
+  tbodyThingDetails: null,
+  crudThings: null,
+  buttonThingDefinitions: null,
+  inputThingDefinition: null,
+  ulThingDefinitions: null,
+};
+
+/**
+ * Initializes components. Should be called after DOMContentLoaded event
+ */
+export async function ready() {
+  Things.addChangeListener(onThingChanged);
+
+  Utils.getAllElementsById(dom);
+
+  thingJsonEditor = Utils.createAceEditor('thingJsonEditor', 'ace/mode/json', true);
+
+  Utils.addDropDownEntries(dom.ulThingDefinitions, Object.keys(thingTemplates));
+
+  dom.ulThingDefinitions.addEventListener('click', onThingDefinitionsClick);
+  dom.crudThings.addEventListener('onCreateClick', onCreateThingClick);
+  dom.crudThings.addEventListener('onUpdateClick', onUpdateThingClick);
+  dom.crudThings.addEventListener('onDeleteClick', onDeleteThingClick);
+  dom.crudThings.addEventListener('onEditToggle', onEditToggle);
+
+  document.querySelector('a[data-bs-target="#tabModifyThing"]').addEventListener('shown.bs.tab', (event) => {
+    thingJsonEditor.renderer.updateFull();
+  });
+}
+
+function onDeleteThingClick() {
+  Utils.confirm(`Are you sure you want to delete thing
    '${dom.crudThings.idValue}'?`, 'Delete', () => {
+    API.callDittoREST('DELETE', `/things/${dom.crudThings.idValue}`, null,
+        {
+          'if-match': '*',
+        },
+    ).then(() => {
+      ThingsSearch.performLastSearch();
+    });
+  });
+}
+
+function onUpdateThingClick() {
+  API.callDittoREST('PUT', `/things/${dom.crudThings.idValue}`, JSON.parse(thingJsonEditor.getValue()),
+      {
+        'if-match': eTag,
+      },
+  ).then(() => {
+    dom.crudThings.toggleEdit();
+    Things.refreshThing(dom.crudThings.idValue, null);
+  });
+}
+
+async function onCreateThingClick() {
+  const editorValue = thingJsonEditor.getValue();
+  if (dom.crudThings.idValue !== undefined && dom.crudThings.idValue !== '') {
+    API.callDittoREST('PUT',
+        '/things/' + dom.crudThings.idValue,
+        editorValue === '' ? {} : JSON.parse(editorValue),
+        {
+          'if-none-match': '*',
+        },
+    ).then((data) => {
+      dom.crudThings.toggleEdit();
+      Things.refreshThing(data.thingId, () => {
+        ThingsSearch.getThings([data.thingId]);
+      });
+    });
+  } else {
+    API.callDittoREST('POST', '/things', editorValue === '' ? {} : JSON.parse(editorValue))
+        .then((data) => {
+          dom.crudThings.toggleEdit();
+          Things.refreshThing(data.thingId, () => {
+            ThingsSearch.getThings([data.thingId]);
+          });
+        });
+  }
+}
+
+function onThingDefinitionsClick(event) {
+  Things.setTheThing(null);
+  // isEditing = true;
+  dom.inputThingDefinition.value = event.target.textContent;
+  thingJsonEditor.setValue(JSON.stringify(thingTemplates[event.target.textContent], null, 2), -1);
+}
+
+/**
+ * Update UI components for the given Thing
+ * @param {Object} thingJson Thing json
+ */
+function onThingChanged(thingJson) {
+  if (dom.crudThings.isEditing) {
+    return;
+  }
+
+  updateThingDetailsTable();
+  updateThingJsonEditor();
+
+  function updateThingDetailsTable() {
+    dom.tbodyThingDetails.innerHTML = '';
+    if (thingJson) {
+      Utils.addTableRow(dom.tbodyThingDetails, 'thingId', false, true, thingJson.thingId);
+      Utils.addTableRow(dom.tbodyThingDetails, 'policyId', false, true, thingJson.policyId);
+      Utils.addTableRow(dom.tbodyThingDetails, 'definition', false, true, thingJson.definition ?? '');
+      Utils.addTableRow(dom.tbodyThingDetails, 'revision', false, true, thingJson._revision);
+      Utils.addTableRow(dom.tbodyThingDetails, 'created', false, true, thingJson._created);
+      Utils.addTableRow(dom.tbodyThingDetails, 'modified', false, true, thingJson._modified);
+    }
+  }
+
+  function updateThingJsonEditor() {
+    if (thingJson) {
+      dom.crudThings.idValue = thingJson.thingId;
+      dom.inputThingDefinition.value = thingJson.definition ?? '';
+      const thingCopy = JSON.parse(JSON.stringify(thingJson));
+      delete thingCopy['_revision'];
+      delete thingCopy['_created'];
+      delete thingCopy['_modified'];
+      thingJsonEditor.setValue(JSON.stringify(thingCopy, null, 2), -1);
+      thingJsonEditor.session.getUndoManager().reset();
+    } else {
+      dom.crudThings.idValue = null;
+      dom.inputThingDefinition.value = null;
+      thingJsonEditor.setValue('');
+      thingJsonEditor.session.getUndoManager().reset();
+    }
+  }
+}
+
+function onEditToggle(event) {
+  const isEditing = event.detail.isEditing;
+  if (isEditing && Things.theThing) {
+    API.callDittoREST('GET', `/things/${Things.theThing.thingId}`, null, null, true)
+        .then((response) => {
+          eTag = response.headers.get('ETag');
+          return response.json();
+        })
+        .then((thingJson) => {
+          enableDisableEditors();
+          initializeEditors(thingJson);
+        });
+  } else {
+    enableDisableEditors();
+    resetEditors();
+  }
+
+  function enableDisableEditors() {
+    dom.buttonThingDefinitions.disabled = !isEditing;
+    dom.inputThingDefinition.disabled = !isEditing;
+    thingJsonEditor.setReadOnly(!isEditing);
+    thingJsonEditor.renderer.setShowGutter(isEditing);
+  }
+
+  function initializeEditors(thingJson) {
+    dom.inputThingDefinition.value = thingJson.definition ?? '';
+    thingJsonEditor.setValue(JSON.stringify(thingJson, null, 2), -1);
+  }
+
+  function resetEditors() {
+    if (dom.crudThings.idValue && dom.crudThings.idValue !== '') {
+      Things.refreshThing(dom.crudThings.idValue, null);
+    } else {
+      dom.inputThingDefinition.value = null;
+      thingJsonEditor.setValue('');
+    }
+  }
+}
+
+
diff --git a/ui/modules/things/thingsSSE.js b/ui/modules/things/thingsSSE.js
new file mode 100644
index 00000000000..b4145e95dc1
--- /dev/null
+++ b/ui/modules/things/thingsSSE.js
@@ -0,0 +1,92 @@
+/*
+* Copyright (c) 2022 Contributors to the Eclipse Foundation
+*
+* See the NOTICE file(s) distributed with this work for additional
+* information regarding copyright ownership.
+*
+* This program and the accompanying materials are made available under the
+* terms of the Eclipse Public License 2.0 which is available at
+* http://www.eclipse.org/legal/epl-2.0
+*
+* SPDX-License-Identifier: EPL-2.0
+*/
+/* eslint-disable require-jsdoc */
+/* eslint-disable arrow-parens */
+// @ts-check
+import * as API from '../api.js';
+import * as Environments from '../environments/environments.js';
+
+import * as Things from './things.js';
+import * as ThingsSearch from './thingsSearch.js';
+import merge from 'lodash/merge';
+
+let selectedThingEventSource;
+let thingsTableEventSource;
+
+
+/**
+ * Initializes components. Should be called after DOMContentLoaded event
+ */
+export async function ready() {
+  Things.addChangeListener(onSelectedThingChanged);
+  ThingsSearch.addChangeListener(onThingsTableChanged);
+  Environments.addChangeListener(onEnvironmentChanged);
+}
+
+const observers = [];
+export function addChangeListener(observer) {
+  observers.push(observer);
+}
+function notifyAll(thingJson) {
+  observers.forEach(observer => observer.call(null, thingJson));
+}
+
+function onThingsTableChanged(thingIds, fieldsQueryParameter) {
+  stopSSE(thingsTableEventSource);
+  if (thingIds && thingIds.length > 0) {
+    console.log('SSE Start: THINGS TABLE');
+    thingsTableEventSource = API.getEventSource(thingIds, fieldsQueryParameter);
+    thingsTableEventSource.onmessage = onMessageThingsTable;
+  }
+}
+
+function onSelectedThingChanged(newThingJson, isNewThingId) {
+  if (!newThingJson) {
+    stopSSE(selectedThingEventSource);
+  } else if (isNewThingId) {
+    selectedThingEventSource && selectedThingEventSource.close();
+    console.log('SSE Start: SELECTED THING : ' + newThingJson.thingId);
+    selectedThingEventSource = API.getEventSource(newThingJson.thingId,
+        'fields=thingId,attributes,features,_revision,_modified');
+    selectedThingEventSource.onmessage = onMessageSelectedThing;
+  }
+}
+
+function stopSSE(eventSource) {
+  if (eventSource) {
+    eventSource.close();
+    console.log('SSE Stopped: ' + (eventSource === selectedThingEventSource ? 'SELECTED THING' : 'THINGS TABLE'));
+  }
+}
+
+function onEnvironmentChanged(modifiedField) {
+  if (!['pinnedThings', 'filterList', 'messageTemplates'].includes(modifiedField)) {
+    stopSSE(selectedThingEventSource);
+    stopSSE(thingsTableEventSource);
+  }
+}
+
+function onMessageSelectedThing(event) {
+  if (event.data && event.data !== '') {
+    const merged = merge(Things.theThing, JSON.parse(event.data));
+    Things.setTheThing(merged);
+    notifyAll(JSON.parse(event.data));
+  }
+}
+
+function onMessageThingsTable(event) {
+  if (event.data && event.data !== '') {
+    ThingsSearch.updateTableRow(JSON.parse(event.data));
+  }
+}
+
diff --git a/ui/modules/things/thingsSearch.js b/ui/modules/things/thingsSearch.js
new file mode 100644
index 00000000000..1f0726cc505
--- /dev/null
+++ b/ui/modules/things/thingsSearch.js
@@ -0,0 +1,312 @@
+/*
+* Copyright (c) 2022 Contributors to the Eclipse Foundation
+*
+* See the NOTICE file(s) distributed with this work for additional
+* information regarding copyright ownership.
+*
+* This program and the accompanying materials are made available under the
+* terms of the Eclipse Public License 2.0 which is available at
+* http://www.eclipse.org/legal/epl-2.0
+*
+* SPDX-License-Identifier: EPL-2.0
+*/
+
+/* eslint-disable require-jsdoc */
+/* eslint-disable new-cap */
+/* eslint-disable no-invalid-this */
+/* eslint-disable arrow-parens */
+
+// @ts-check
+
+// @ts-ignore
+import {JSONPath} from 'jsonpath-plus';
+
+import * as API from '../api.js';
+
+import * as Utils from '../utils.js';
+import * as Fields from './fields.js';
+import * as Things from './things.js';
+import * as ThingsSSE from './thingsSSE.js';
+import * as Environments from '../environments/environments.js';
+
+let lastSearch = '';
+let theSearchCursor;
+
+const dom = {
+  thingsTableHead: null,
+  thingsTableBody: null,
+  searchFilterEdit: null,
+  favIcon: null,
+};
+
+const observers = [];
+export function addChangeListener(observer) {
+  observers.push(observer);
+}
+function notifyAll(thingIds, fields) {
+  observers.forEach(observer => observer.call(null, thingIds, fields));
+}
+
+
+export async function ready() {
+  Things.addChangeListener(onThingChanged);
+  ThingsSSE.addChangeListener(updateTableRow);
+
+  Utils.getAllElementsById(dom);
+
+  dom.thingsTableBody.addEventListener('click', onThingsTableClicked);
+}
+
+function onThingsTableClicked(event) {
+  if (event.target && event.target.nodeName === 'TD') {
+    const row = event.target.parentNode;
+    if (row.id === 'searchThingsMore') {
+      row.style.pointerEvents = 'none';
+      event.stopImmediatePropagation();
+      searchThings(dom.searchFilterEdit.value, true);
+    } else {
+      if (Things.theThing && Things.theThing.thingId === row.id) {
+        event.stopImmediatePropagation();
+        Things.setTheThing(null);
+      } else {
+        Things.refreshThing(row.id, null);
+      }
+    }
+  }
+}
+
+/**
+ * Tests if the search filter is an RQL. If yes, things search is called otherwise just things get
+ * @param {String} filter search filter string containing an RQL or a thingId
+ */
+export function searchTriggered(filter) {
+  lastSearch = filter;
+  const regex = /^(eq\(|ne\(|gt\(|ge\(|lt\(|le\(|in\(|like\(|exists\(|and\(|or\(|not\().*/;
+  if (filter === '' || regex.test(filter)) {
+    searchThings(filter);
+  } else {
+    getThings([filter]);
+  }
+}
+
+/**
+ * Gets the list of pinned things
+ */
+export function pinnedTriggered() {
+  lastSearch = 'pinned';
+  dom.searchFilterEdit.value = null;
+  dom.favIcon.classList.replace('bi-star-fill', 'bi-star');
+  getThings(Environments.current()['pinnedThings']);
+}
+
+/**
+ * Performs the last search by the user using the last used filter.
+ * If the user used pinned things last time, the pinned things are reloaded
+ */
+export function performLastSearch() {
+  if (lastSearch === 'pinned') {
+    pinnedTriggered();
+  } else {
+    searchTriggered(lastSearch);
+  }
+}
+
+/**
+ * Gets things from Ditto by thingIds and fills the UI with the result
+ * @param {Array} thingIds Array of thingIds
+ */
+export function getThings(thingIds) {
+  dom.thingsTableBody.innerHTML = '';
+  const fieldsQueryParameter = Fields.getQueryParameter();
+  if (thingIds.length > 0) {
+    API.callDittoREST('GET',
+        `/things?${fieldsQueryParameter}&ids=${thingIds}&option=sort(%2BthingId)`)
+        .then((thingJsonArray) => {
+          fillThingsTable(thingJsonArray);
+          notifyAll(thingIds, fieldsQueryParameter);
+        })
+        .catch((error) => {
+          resetAndClearViews();
+          notifyAll(null);
+        });
+  } else {
+    resetAndClearViews();
+    notifyAll(null);
+  }
+}
+
+function resetAndClearViews(retainThing = false) {
+  theSearchCursor = null;
+  dom.thingsTableHead.innerHTML = '';
+  dom.thingsTableBody.innerHTML = '';
+  if (!retainThing) {
+    Things.setTheThing(null);
+  }
+}
+
+/**
+ * Calls Ditto search api and fills UI with the result
+ * @param {String} filter Ditto search filter (rql)
+ * @param {boolean} isMore (optional) use cursor from previous search for additional pages
+ */
+function searchThings(filter, isMore = false) {
+  document.body.style.cursor = 'progress';
+
+  const namespaces = Environments.current().searchNamespaces;
+  const fieldsQueryParameter = Fields.getQueryParameter();
+  API.callDittoREST('GET',
+      '/search/things?' + fieldsQueryParameter +
+      ((filter && filter !== '') ? '&filter=' + encodeURIComponent(filter) : '') +
+      ((namespaces && namespaces !== '') ? '&namespaces=' + namespaces : '') +
+      '&option=sort(%2BthingId)' +
+      // ',size(3)' +
+      (isMore ? ',cursor(' + theSearchCursor + ')' : ''),
+  ).then((searchResult) => {
+    if (isMore) {
+      removeMoreFromThingList();
+    } else {
+      resetAndClearViews(true);
+    }
+    fillThingsTable(searchResult.items);
+    checkMorePages(searchResult);
+    if (!isMore) {
+      notifyAll(searchResult.items.map(thingJson => thingJson.thingId), fieldsQueryParameter);
+    }
+  }).catch((error) => {
+    resetAndClearViews();
+    notifyAll(null);
+  }).finally(() => {
+    document.body.style.cursor = 'default';
+  });
+
+  function checkMorePages(searchResult) {
+    if (searchResult['cursor']) {
+      addMoreToThingList();
+      theSearchCursor = searchResult.cursor;
+    } else {
+      theSearchCursor = null;
+    }
+  }
+
+  function addMoreToThingList() {
+    const moreCell = dom.thingsTableBody.insertRow().insertCell(-1);
+    moreCell.innerHTML = 'load more...';
+    moreCell.colSpan = dom.thingsTableBody.rows[0].childElementCount;
+    moreCell.style.textAlign = 'center';
+    moreCell.style.cursor = 'pointer';
+    moreCell.disabled = true;
+    moreCell.style.color = '#3a8c9a';
+    moreCell.parentNode.id = 'searchThingsMore';
+  }
+}
+
+/**
+ * remove the "more" line from the things table
+ */
+export function removeMoreFromThingList() {
+  const moreRow = document.getElementById('searchThingsMore');
+  if (moreRow) {
+    moreRow.parentNode.removeChild(moreRow);
+  }
+}
+
+
+/**
+ * Fills the things table UI with the given things
+ * @param {Array} thingsList Array of thing json objects
+ */
+function fillThingsTable(thingsList) {
+  const activeFields = Environments.current().fieldList.filter((f) => f.active);
+  fillHeaderRow();
+  let thingSelected = false;
+  thingsList.forEach((item, t) => {
+    const row = dom.thingsTableBody.insertRow();
+    fillBodyRow(row, item);
+  });
+  if (!thingSelected) {
+    Things.setTheThing(null);
+  }
+
+  function fillHeaderRow() {
+    dom.thingsTableHead.innerHTML = '';
+    // Utils.addCheckboxToRow(dom.thingsTableHead, 'checkboxHead', false, null);
+    Utils.insertHeaderCell(dom.thingsTableHead, '');
+    Utils.insertHeaderCell(dom.thingsTableHead, 'Thing ID');
+    activeFields.forEach((field) => {
+      Utils.insertHeaderCell(dom.thingsTableHead, field['label'] ? field.label : field.path);
+    });
+  }
+
+  function fillBodyRow(row, item) {
+    row.id = item.thingId;
+    if (Things.theThing && (item.thingId === Things.theThing.thingId)) {
+      thingSelected = true;
+      row.classList.add('table-active');
+    }
+    Utils.addCheckboxToRow(
+        row,
+        item.thingId,
+        Environments.current().pinnedThings.includes(item.thingId),
+        togglePinnedThing,
+    );
+    Utils.addCellToRow(row, beautifyId(item.thingId), item.thingId);
+    activeFields.forEach((field) => {
+      let path = field.path.replace(/\//g, '.');
+      if (path.charAt(0) !== '.') {
+        path = '$.' + path;
+      }
+      const elem = JSONPath({
+        json: item,
+        path: path,
+      });
+      Utils.addCellToRow(row, elem.length !== 0 ? elem[0] : '').setAttribute('jsonPath', path);
+    });
+  }
+
+  function beautifyId(longId) {
+    const uuidRegex = /([0-9a-f]{7})[0-9a-f]-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;
+    let result = longId;
+    if (Environments.current()['shortenUUID']) {
+      result = result.replace(uuidRegex, '$1');
+    }
+    if (Environments.current()['defaultNamespace']) {
+      result = result.replace(Environments.current()['defaultNamespace'], 'dn');
+    }
+    return result;
+  }
+}
+
+function togglePinnedThing(evt) {
+  if (evt.target.checked) {
+    Environments.current().pinnedThings.push(this.id);
+  } else {
+    const index = Environments.current().pinnedThings.indexOf(this.id);
+    if (index > -1) {
+      Environments.current().pinnedThings.splice(index, 1);
+    }
+  }
+  Environments.environmentsJsonChanged('pinnedThings');
+}
+
+function onThingChanged(thingJson) {
+  if (!thingJson) {
+    Utils.tableAdjustSelection(dom.thingsTableBody, () => false);
+  }
+}
+
+export function updateTableRow(thingUpdateJson) {
+  const row = document.getElementById(thingUpdateJson.thingId);
+  console.assert(row !== null, 'Unexpected thingId for table update. thingId was not loaded before');
+  Array.from(row.cells).forEach((cell) => {
+    const path = cell.getAttribute('jsonPath');
+    if (path) {
+      const elem = JSONPath({
+        json: thingUpdateJson,
+        path: path,
+      });
+      if (elem.length !== 0) {
+        cell.innerHTML = elem[0];
+      }
+    }
+  });
+}
diff --git a/ui/modules/things/wotDescription.js b/ui/modules/things/wotDescription.js
index 7e099772b75..7b2077df19e 100644
--- a/ui/modules/things/wotDescription.js
+++ b/ui/modules/things/wotDescription.js
@@ -15,20 +15,21 @@
 import * as API from '../api.js';
 import * as Utils from '../utils.js';
 import * as Things from './things.js';
+import wotDescriptionHTML from './wotDescription.html';
 
 export function WoTDescription(targetTab, forFeature) {
   let tabLink;
   let aceWoTDescription;
   let viewDirty = false;
   const _forFeature = forFeature;
-  const domTheFeatureId = document.getElementById('theFeatureId');
+  let theFeatureId;
 
   const ready = async () => {
     const tabId = Utils.addTab(
         document.getElementById(targetTab.itemsId),
         document.getElementById(targetTab.contentId),
         'WoT TD',
-        await( await fetch('modules/things/wotDescription.html')).text(),
+        wotDescriptionHTML,
         'Generated WoT Thing Description. This requires a valid reference to a WoT Thing Model in the ' +
         (forFeature ? 'Feature' : 'Thing') +
         ' definition. The Ditto environment must have WoT support enabled.'
@@ -43,8 +44,11 @@ export function WoTDescription(targetTab, forFeature) {
     aceWoTDescription = Utils.createAceEditor(aceId, 'ace/mode/json', true);
   };
 
-  const onReferenceChanged = () => {
-    if (tabLink.classList.contains('active')) {
+  const onReferenceChanged = (ref) => {
+    if (_forFeature) {
+      theFeatureId = ref;
+    }
+    if (tabLink && tabLink.classList.contains('active')) {
       refreshDescription();
     } else {
       viewDirty = true;
@@ -64,12 +68,10 @@ export function WoTDescription(targetTab, forFeature) {
   }
 
   function refreshDescription() {
-    let featurePath = '';
-    if (_forFeature) {
-      featurePath = '/features/' + domTheFeatureId.value;
-    }
+    const featurePath = _forFeature ? '/features/' + theFeatureId : '';
+
     aceWoTDescription.setValue('');
-    if (Things.theThing && (!_forFeature || domTheFeatureId.value)) {
+    if (Things.theThing && (!_forFeature || theFeatureId)) {
       API.callDittoREST(
           'GET',
           '/things/' + Things.theThing.thingId + featurePath,
diff --git a/ui/modules/utils.js b/ui/modules/utils.js
index 9f5d7808540..d1f286e9b7e 100644
--- a/ui/modules/utils.js
+++ b/ui/modules/utils.js
@@ -12,6 +12,9 @@
  */
 
 /* eslint-disable quotes */
+import autoComplete from "@tarekraafat/autocomplete.js";
+import {Toast, Modal} from 'bootstrap';
+
 
 const dom = {
   modalBodyConfirm: null,
@@ -28,11 +31,12 @@ export function ready() {
 
 /**
  * Adds a table to a table element
- * @param {HTMLElement} table tbody element the row is added to
+ * @param {HTMLTableElement} table tbody element the row is added to
  * @param {String} key first column text of the row. Acts as id of the row
  * @param {boolean} selected if true, the new row will be marked as selected
  * @param {boolean} withClipBoardCopy add a clipboard button at the last column of the row
  * @param {array} columnValues texts for additional columns of the row
+ * @return {Element} created row
  */
 export const addTableRow = function(table, key, selected, withClipBoardCopy, ...columnValues) {
   const row = table.insertRow();
@@ -47,6 +51,7 @@ export const addTableRow = function(table, key, selected, withClipBoardCopy, ...
   if (withClipBoardCopy) {
     addClipboardCopyToRow(row);
   }
+  return row;
 };
 
 /**
@@ -69,16 +74,18 @@ export function addCheckboxToRow(row, id, checked, onToggle) {
 
 /**
  * Adds a cell to the row including a tooltip
- * @param {HTMRTableRowElement} row target row
+ * @param {HTMLTableRowElement} row target row
  * @param {String} cellContent content of new cell
  * @param {String} cellTooltip tooltip for new cell
- * @param {integer} position optional, default -1 (add to the end)
+ * @param {Number} position optional, default -1 (add to the end)
+ * @return {HTMLElement} created cell element 
  */
-export function addCellToRow(row, cellContent, cellTooltip, position) {
-  const cell = row.insertCell(position ?? -1);
+export function addCellToRow(row, cellContent, cellTooltip = null, position = -1) {
+  const cell = row.insertCell(position);
   cell.innerHTML = cellContent;
   cell.setAttribute('data-bs-toggle', 'tooltip');
   cell.title = cellTooltip ?? cellContent;
+  return cell;
 }
 
 /**
@@ -149,12 +156,12 @@ export function setOptions(target, options) {
  * @param {array} items array of items for the drop down
  * @param {boolean} isHeader (optional) true to add a header line
  */
-export function addDropDownEntries(target, items, isHeader) {
+export function addDropDownEntries(target, items, isHeader = false) {
   items.forEach((value) => {
     const li = document.createElement('li');
     li.innerHTML = isHeader ?
         `
    ${value}
    ` :
-        `${value}`;
+        `${value}`;
     target.appendChild(li);
   });
 }
@@ -191,10 +198,11 @@ export function addTab(tabItemsNode, tabContentsNode, title, contentHTML, toolTi
 /**
  * Get the HTMLElements of all the given ids. The HTMLElements will be returned in the original object
  * @param {Object} domObjects object with empty keys that are used as ids of the dom elements
+ * @param {Element} searchRoot optional root to search in (optional for shadow dom)
  */
-export function getAllElementsById(domObjects) {
+export function getAllElementsById(domObjects, searchRoot = document) {
   Object.keys(domObjects).forEach((id) => {
-    domObjects[id] = document.getElementById(id);
+    domObjects[id] = searchRoot.getElementById(id);
     if (!domObjects[id]) {
       throw new Error(`Element ${id} not found.`);
     }
@@ -207,13 +215,13 @@ export function getAllElementsById(domObjects) {
  * @param {String} header Header for toast
  * @param {String} status Status text for toas
  */
-export function showError(message, header, status) {
+export function showError(message, header, status = '') {
   const domToast = document.createElement('div');
   domToast.classList.add('toast');
   domToast.innerHTML = `
    
   
   ${header ?? 'Error'}
-  ${status ?? ''}
+  ${status}
   
   
    
   
    ${message}
    `;
@@ -222,7 +230,7 @@ export function showError(message, header, status) {
   domToast.addEventListener("hidden.bs.toast", () => {
     domToast.remove();
   });
-  const bsToast = new bootstrap.Toast(domToast);
+  const bsToast = new Toast(domToast);
   bsToast.show();
 }
 
@@ -258,16 +266,16 @@ export function assert(condition, message, validatedElement) {
 }
 
 /**
- * Simple Date format that makes UTC string more readable and cuts off the milliseconds
- * @param {Date} date to format
+ * Simple Date format that makes ISO string more readable and cuts off the milliseconds
+ * @param {String} dateISOString to format
  * @param {boolean} withMilliseconds don t cut off milliseconds if true
  * @return {String} formatted date
  */
-export function formatDate(date, withMilliseconds) {
+export function formatDate(dateISOString, withMilliseconds) {
   if (withMilliseconds) {
-    return date.replace('T', ' ').replace('Z', '').replace('.', ' ');
+    return dateISOString.replace('T', ' ').replace('Z', '').replace('.', ' ');
   } else {
-    return date.split('.')[0].replace('T', ' ');
+    return dateISOString.split('.')[0].replace('T', ' ');
   }
 }
 
@@ -280,7 +288,7 @@ let modalConfirm;
  * @param {function} callback true if confirmed
  */
 export function confirm(message, action, callback) {
-  modalConfirm = modalConfirm ?? new bootstrap.Modal('#modalConfirm');
+  modalConfirm = modalConfirm ?? new Modal('#modalConfirm');
   dom.modalBodyConfirm.innerHTML = message;
   dom.buttonConfirmed.innerText = action;
   dom.buttonConfirmed.onclick = callback;
@@ -306,6 +314,52 @@ export function createAceEditor(domId, sessionMode, readOnly) {
   return result;
 }
 
+/**
+ * Creates a autocomplete input field
+ * @param {String} selector selector for the input field
+ * @param {function} src src
+ * @param {String} placeHolder placeholder for input field
+ * @return {Object} autocomplete instance
+ */
+export function createAutoComplete(selector, src, placeHolder) {
+  return new autoComplete({
+    selector: selector,
+    data: {
+      src: src,
+      keys: ['label', 'group'],
+    },
+    placeHolder: placeHolder,
+    resultsList: {
+      class: 'dropdown-menu show',
+      maxResults: 30,
+    },
+    resultItem: {
+      class: 'dropdown-item',
+      highlight: true,
+      element: (item, data) => {
+        item.style = 'display: flex;';
+        item.innerHTML = `${data.key === 'label' ? data.match : data.value.label}
+            
+              ${data.key === 'group' ? data.match : data.value.group}`;
+      },
+    },
+    events: {
+      input: {
+        results: () => {
+          Array.from(document.getElementsByClassName('resizable_pane')).forEach((resizePane) => {
+            resizePane.style.overflow = 'unset';
+          });
+        },
+        close: () => {
+          Array.from(document.getElementsByClassName('resizable_pane')).forEach((resizePane) => {
+            resizePane.style.overflow = 'auto';
+          });
+        },
+      },
+    },
+  });
+}
+
 /**
  * Links the hidden input element for validation to the table
  * @param {HTMLElement} tableElement tbody that is validated
@@ -319,7 +373,7 @@ export function addValidatorToTable(tableElement, inputElement) {
 
 /**
  * Adjust selection of a table
- * @param {HTMLElement} tbody table with the data
+ * @param {HTMLTableElement} tbody table with the data
  * @param {function} condition evaluate if table row should be selected or not
  */
 export function tableAdjustSelection(tbody, condition) {
diff --git a/ui/modules/utils/crudToolbar.html b/ui/modules/utils/crudToolbar.html
new file mode 100644
index 00000000000..5a9c3f601fd
--- /dev/null
+++ b/ui/modules/utils/crudToolbar.html
@@ -0,0 +1,29 @@
+
+
+
+
    
+  
    
+    
+    
+    
+    
+    
+    
+    
+    
    
+  
    
+  
+
    
\ No newline at end of file
diff --git a/ui/modules/utils/crudToolbar.js b/ui/modules/utils/crudToolbar.js
new file mode 100644
index 00000000000..a7c300e30d4
--- /dev/null
+++ b/ui/modules/utils/crudToolbar.js
@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2022 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+/* eslint-disable require-jsdoc */
+import * as Utils from '../utils.js';
+import crudToolbarHTML from './crudToolbar.html';
+
+class CrudToolbar extends HTMLElement {
+  isEditing = false;
+  isEditDisabled = false;
+  isCreateDisabled = false;
+  isDeleteDisabled = false;
+  dom = {
+    label: null,
+    inputIdValue: null,
+    buttonEdit: null,
+    buttonCreate: null,
+    buttonUpdate: null,
+    buttonDelete: null,
+    buttonCancel: null,
+    divRoot: null,
+  };
+
+  get idValue() {
+    return this.dom.inputIdValue.value;
+  }
+
+  set idValue(newValue) {
+    this.shadowRoot.getElementById('inputIdValue').value = newValue;
+    const buttonDelete = this.shadowRoot.getElementById('buttonDelete');
+    if (!this.isDeleteDisabled && newValue && newValue !== '') {
+      buttonDelete.removeAttribute('hidden');
+    } else {
+      buttonDelete.setAttribute('hidden', '');
+    }
+  }
+
+  get editDisabled() {
+    return this.isEditDisabled;
+  }
+
+  set createDisabled(newValue) {
+    this.isCreateDisabled = newValue;
+    this.setButtonState('buttonCreate', newValue);
+  }
+
+  set deleteDisabled(newValue) {
+    this.isDeleteDisabled = newValue;
+    this.setButtonState('buttonDelete', newValue);
+  }
+
+  set editDisabled(newValue) {
+    this.isEditDisabled = newValue;
+    if (!this.isEditing) {
+      this.setButtonState('buttonEdit', newValue);
+    }
+  }
+
+  setButtonState(buttonId, isDisabled) {
+    const button = this.shadowRoot.getElementById(buttonId);
+
+    if (isDisabled) {
+      button.setAttribute('hidden', '');
+    } else {
+      button.removeAttribute('hidden');
+    }
+  }
+  get validationElement() {
+    return this.dom.inputIdValue;
+  }
+
+  constructor() {
+    super();
+    this.attachShadow({mode: 'open'});
+  }
+
+  connectedCallback() {
+    this.shadowRoot.innerHTML = crudToolbarHTML;
+
+    setTimeout(() => {
+      Utils.getAllElementsById(this.dom, this.shadowRoot);
+
+      this.dom.buttonEdit.onclick = () => this.toggleEdit(false);
+      this.dom.buttonCancel.onclick = () => this.toggleEdit(true);
+      this.dom.label.innerText = this.getAttribute('label') || 'Label';
+      this.dom.buttonCreate.onclick = this.eventDispatcher('onCreateClick');
+      this.dom.buttonUpdate.onclick = this.eventDispatcher('onUpdateClick');
+      this.dom.buttonDelete.onclick = this.eventDispatcher('onDeleteClick');
+    });
+  };
+
+  eventDispatcher(eventName) {
+    return () => {
+      this.dispatchEvent(new CustomEvent(eventName, {
+        composed: true,
+      }));
+    };
+  }
+
+  toggleEdit(isCancel) {
+    this.isEditing = !this.isEditing;
+    document.getElementById('modalCrudEdit').classList.toggle('editBackground');
+    this.dom.divRoot.classList.toggle('editForground');
+
+    if (this.isEditing || this.isEditDisabled) {
+      this.dom.buttonEdit.setAttribute('hidden', '');
+    } else {
+      this.dom.buttonEdit.removeAttribute('hidden');
+    }
+    this.dom.buttonCancel.toggleAttribute('hidden');
+
+    if (this.isEditing) {
+      if (this.dom.inputIdValue.value) {
+        this.dom.buttonUpdate.toggleAttribute('hidden');
+      } else if (!this.isCreateDisabled) {
+        this.dom.buttonCreate.toggleAttribute('hidden');
+      }
+    } else {
+      this.dom.buttonCreate.setAttribute('hidden', '');
+      this.dom.buttonUpdate.setAttribute('hidden', '');
+    }
+    if (this.isEditing || !this.dom.inputIdValue.value) {
+      this.dom.buttonDelete.setAttribute('hidden', '');
+    }
+    const allowIdChange = this.isEditing && (!this.dom.inputIdValue.value || this.hasAttribute('allowIdChange'));
+    this.dom.inputIdValue.disabled = !allowIdChange;
+    this.dispatchEvent(new CustomEvent('onEditToggle', {
+      composed: true,
+      detail: {
+        isEditing: this.isEditing,
+        isCancel: isCancel,
+      },
+    }));
+  }
+}
+
+customElements.define('crud-toolbar', CrudToolbar);
diff --git a/ui/modules/utils/tabHandler.js b/ui/modules/utils/tabHandler.js
new file mode 100644
index 00000000000..be7b5a4f746
--- /dev/null
+++ b/ui/modules/utils/tabHandler.js
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+/* eslint-disable require-jsdoc */
+import * as Environments from '../environments/environments.js';
+import * as Authorization from '../environments/authorization.js';
+
+export function TabHandler(domTabItem, domTabContent, onRefreshTab, envDisabledKey) {
+  const _domTabItem = domTabItem;
+  const _domTabContent = domTabContent;
+  const _refreshTab = onRefreshTab;
+  const _envDisabledKey = envDisabledKey;
+  let viewDirty = false;
+
+  _domTabItem.addEventListener('click', onTabActivated);
+  Environments.addChangeListener(onEnvironmentChanged);
+
+  return {
+    set viewDirty(newValue) {
+      viewDirty = newValue;
+    },
+  };
+
+  function onTabActivated() {
+    Authorization.setForDevops(_domTabItem.dataset.auth === 'devOps');
+
+    if (viewDirty) {
+      _refreshTab.call(null, false);
+      viewDirty = false;
+    }
+  }
+
+  function onEnvironmentChanged(modifiedField) {
+    _domTabItem.toggleAttribute('hidden', Environments.current()[_envDisabledKey] ?? false);
+    if (!['pinnedThings', 'filterList', 'messageTemplates'].includes(modifiedField)) {
+      if (_domTabContent.classList.contains('show')) {
+        _refreshTab.call(null, true);
+      } else {
+        viewDirty = true;
+      }
+    }
+  }
+}
+
+
diff --git a/ui/package-lock.json b/ui/package-lock.json
new file mode 100644
index 00000000000..def1fa08483
--- /dev/null
+++ b/ui/package-lock.json
@@ -0,0 +1,3008 @@
+{
+  "name": "ui",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "ui",
+      "version": "1.0.0",
+      "license": "EPL-2.0",
+      "dependencies": {
+        "@popperjs/core": "^2.11.7",
+        "@tarekraafat/autocomplete.js": "^10.2.7",
+        "bootstrap": "^5.2.3",
+        "bootstrap-icons": "^1.10.5",
+        "event-source-polyfill": "^1.0.31",
+        "jsonpath-plus": "^7.2.0",
+        "lodash": "^4.17.21"
+      },
+      "devDependencies": {
+        "esbuild": "^0.17.18",
+        "esbuild-sass-plugin": "^2.9.0",
+        "eslint": "^8.40.0",
+        "eslint-config-google": "^0.14.0",
+        "eslint-config-standard": "^17.0.0",
+        "eslint-plugin-import": "^2.27.5",
+        "eslint-plugin-n": "^15.7.0",
+        "eslint-plugin-promise": "^6.1.1"
+      }
+    },
+    "node_modules/@esbuild/android-arm": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.18.tgz",
+      "integrity": "sha512-EmwL+vUBZJ7mhFCs5lA4ZimpUH3WMAoqvOIYhVQwdIgSpHC8ImHdsRyhHAVxpDYUSm0lWvd63z0XH1IlImS2Qw==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/android-arm64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.18.tgz",
+      "integrity": "sha512-/iq0aK0eeHgSC3z55ucMAHO05OIqmQehiGay8eP5l/5l+iEr4EIbh4/MI8xD9qRFjqzgkc0JkX0LculNC9mXBw==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/android-x64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.18.tgz",
+      "integrity": "sha512-x+0efYNBF3NPW2Xc5bFOSFW7tTXdAcpfEg2nXmxegm4mJuVeS+i109m/7HMiOQ6M12aVGGFlqJX3RhNdYM2lWg==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/darwin-arm64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.18.tgz",
+      "integrity": "sha512-6tY+djEAdF48M1ONWnQb1C+6LiXrKjmqjzPNPWXhu/GzOHTHX2nh8Mo2ZAmBFg0kIodHhciEgUBtcYCAIjGbjQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/darwin-x64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.18.tgz",
+      "integrity": "sha512-Qq84ykvLvya3dO49wVC9FFCNUfSrQJLbxhoQk/TE1r6MjHo3sFF2tlJCwMjhkBVq3/ahUisj7+EpRSz0/+8+9A==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/freebsd-arm64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.18.tgz",
+      "integrity": "sha512-fw/ZfxfAzuHfaQeMDhbzxp9mc+mHn1Y94VDHFHjGvt2Uxl10mT4CDavHm+/L9KG441t1QdABqkVYwakMUeyLRA==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/freebsd-x64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.18.tgz",
+      "integrity": "sha512-FQFbRtTaEi8ZBi/A6kxOC0V0E9B/97vPdYjY9NdawyLd4Qk5VD5g2pbWN2VR1c0xhzcJm74HWpObPszWC+qTew==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/linux-arm": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.18.tgz",
+      "integrity": "sha512-jW+UCM40LzHcouIaqv3e/oRs0JM76JfhHjCavPxMUti7VAPh8CaGSlS7cmyrdpzSk7A+8f0hiedHqr/LMnfijg==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/linux-arm64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.18.tgz",
+      "integrity": "sha512-R7pZvQZFOY2sxUG8P6A21eq6q+eBv7JPQYIybHVf1XkQYC+lT7nDBdC7wWKTrbvMXKRaGudp/dzZCwL/863mZQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/linux-ia32": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.18.tgz",
+      "integrity": "sha512-ygIMc3I7wxgXIxk6j3V00VlABIjq260i967Cp9BNAk5pOOpIXmd1RFQJQX9Io7KRsthDrQYrtcx7QCof4o3ZoQ==",
+      "cpu": [
+        "ia32"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/linux-loong64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.18.tgz",
+      "integrity": "sha512-bvPG+MyFs5ZlwYclCG1D744oHk1Pv7j8psF5TfYx7otCVmcJsEXgFEhQkbhNW8otDHL1a2KDINW20cfCgnzgMQ==",
+      "cpu": [
+        "loong64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/linux-mips64el": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.18.tgz",
+      "integrity": "sha512-oVqckATOAGuiUOa6wr8TXaVPSa+6IwVJrGidmNZS1cZVx0HqkTMkqFGD2HIx9H1RvOwFeWYdaYbdY6B89KUMxA==",
+      "cpu": [
+        "mips64el"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/linux-ppc64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.18.tgz",
+      "integrity": "sha512-3dLlQO+b/LnQNxgH4l9rqa2/IwRJVN9u/bK63FhOPB4xqiRqlQAU0qDU3JJuf0BmaH0yytTBdoSBHrb2jqc5qQ==",
+      "cpu": [
+        "ppc64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/linux-riscv64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.18.tgz",
+      "integrity": "sha512-/x7leOyDPjZV3TcsdfrSI107zItVnsX1q2nho7hbbQoKnmoeUWjs+08rKKt4AUXju7+3aRZSsKrJtaRmsdL1xA==",
+      "cpu": [
+        "riscv64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/linux-s390x": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.18.tgz",
+      "integrity": "sha512-cX0I8Q9xQkL/6F5zWdYmVf5JSQt+ZfZD2bJudZrWD+4mnUvoZ3TDDXtDX2mUaq6upMFv9FlfIh4Gfun0tbGzuw==",
+      "cpu": [
+        "s390x"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/linux-x64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.18.tgz",
+      "integrity": "sha512-66RmRsPlYy4jFl0vG80GcNRdirx4nVWAzJmXkevgphP1qf4dsLQCpSKGM3DUQCojwU1hnepI63gNZdrr02wHUA==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/netbsd-x64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.18.tgz",
+      "integrity": "sha512-95IRY7mI2yrkLlTLb1gpDxdC5WLC5mZDi+kA9dmM5XAGxCME0F8i4bYH4jZreaJ6lIZ0B8hTrweqG1fUyW7jbg==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "netbsd"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/openbsd-x64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.18.tgz",
+      "integrity": "sha512-WevVOgcng+8hSZ4Q3BKL3n1xTv5H6Nb53cBrtzzEjDbbnOmucEVcZeGCsCOi9bAOcDYEeBZbD2SJNBxlfP3qiA==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "openbsd"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/sunos-x64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.18.tgz",
+      "integrity": "sha512-Rzf4QfQagnwhQXVBS3BYUlxmEbcV7MY+BH5vfDZekU5eYpcffHSyjU8T0xucKVuOcdCsMo+Ur5wmgQJH2GfNrg==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "sunos"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/win32-arm64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.18.tgz",
+      "integrity": "sha512-Kb3Ko/KKaWhjeAm2YoT/cNZaHaD1Yk/pa3FTsmqo9uFh1D1Rfco7BBLIPdDOozrObj2sahslFuAQGvWbgWldAg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/win32-ia32": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.18.tgz",
+      "integrity": "sha512-0/xUMIdkVHwkvxfbd5+lfG7mHOf2FRrxNbPiKWg9C4fFrB8H0guClmaM3BFiRUYrznVoyxTIyC/Ou2B7QQSwmw==",
+      "cpu": [
+        "ia32"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@esbuild/win32-x64": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.18.tgz",
+      "integrity": "sha512-qU25Ma1I3NqTSHJUOKi9sAH1/Mzuvlke0ioMJRthLXKm7JiSKVwFghlGbDLOO2sARECGhja4xYfRAZNPAkooYg==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@eslint-community/eslint-utils": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
+      "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==",
+      "dev": true,
+      "dependencies": {
+        "eslint-visitor-keys": "^3.3.0"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "peerDependencies": {
+        "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0"
+      }
+    },
+    "node_modules/@eslint-community/regexpp": {
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.5.1.tgz",
+      "integrity": "sha512-Z5ba73P98O1KUYCCJTUeVpja9RcGoMdncZ6T49FCUl2lN38JtCJ+3WgIDBv0AuY4WChU5PmtJmOCTlN6FZTFKQ==",
+      "dev": true,
+      "engines": {
+        "node": "^12.0.0 || ^14.0.0 || >=16.0.0"
+      }
+    },
+    "node_modules/@eslint/eslintrc": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.3.tgz",
+      "integrity": "sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ==",
+      "dev": true,
+      "dependencies": {
+        "ajv": "^6.12.4",
+        "debug": "^4.3.2",
+        "espree": "^9.5.2",
+        "globals": "^13.19.0",
+        "ignore": "^5.2.0",
+        "import-fresh": "^3.2.1",
+        "js-yaml": "^4.1.0",
+        "minimatch": "^3.1.2",
+        "strip-json-comments": "^3.1.1"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/eslint"
+      }
+    },
+    "node_modules/@eslint/js": {
+      "version": "8.40.0",
+      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.40.0.tgz",
+      "integrity": "sha512-ElyB54bJIhXQYVKjDSvCkPO1iU1tSAeVQJbllWJq1XQSmmA4dgFk8CbiBGpiOPxleE48vDogxCtmMYku4HSVLA==",
+      "dev": true,
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      }
+    },
+    "node_modules/@humanwhocodes/config-array": {
+      "version": "0.11.8",
+      "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz",
+      "integrity": "sha512-UybHIJzJnR5Qc/MsD9Kr+RpO2h+/P1GhOwdiLPXK5TWk5sgTdu88bTD9UP+CKbPPh5Rni1u0GjAdYQLemG8g+g==",
+      "dev": true,
+      "dependencies": {
+        "@humanwhocodes/object-schema": "^1.2.1",
+        "debug": "^4.1.1",
+        "minimatch": "^3.0.5"
+      },
+      "engines": {
+        "node": ">=10.10.0"
+      }
+    },
+    "node_modules/@humanwhocodes/module-importer": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
+      "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",
+      "dev": true,
+      "engines": {
+        "node": ">=12.22"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/nzakas"
+      }
+    },
+    "node_modules/@humanwhocodes/object-schema": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",
+      "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
+      "dev": true
+    },
+    "node_modules/@nodelib/fs.scandir": {
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+      "dev": true,
+      "dependencies": {
+        "@nodelib/fs.stat": "2.0.5",
+        "run-parallel": "^1.1.9"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.stat": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+      "dev": true,
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.walk": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+      "dev": true,
+      "dependencies": {
+        "@nodelib/fs.scandir": "2.1.5",
+        "fastq": "^1.6.0"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@popperjs/core": {
+      "version": "2.11.7",
+      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.7.tgz",
+      "integrity": "sha512-Cr4OjIkipTtcXKjAsm8agyleBuDHvxzeBoa1v543lbv1YaIwQjESsVcmjiWiPEbC1FIeHOG/Op9kdCmAmiS3Kw==",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/popperjs"
+      }
+    },
+    "node_modules/@tarekraafat/autocomplete.js": {
+      "version": "10.2.7",
+      "resolved": "https://registry.npmjs.org/@tarekraafat/autocomplete.js/-/autocomplete.js-10.2.7.tgz",
+      "integrity": "sha512-iE+dnXI8/LrTaSORrnNdSyXg/bFCbCpz/R5GUdB3ioW+9PVEhglxNcSDQNeCXtrbRG0kOBFUd4unEiwcmqyn8A==",
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/autocompletejs"
+        },
+        {
+          "type": "liberapay",
+          "url": "https://liberapay.com/TarekRaafat"
+        },
+        {
+          "type": "patreon",
+          "url": "https://patreon.com/TarekRaafat"
+        }
+      ]
+    },
+    "node_modules/@types/json5": {
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
+      "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==",
+      "dev": true
+    },
+    "node_modules/acorn": {
+      "version": "8.8.2",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz",
+      "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==",
+      "dev": true,
+      "bin": {
+        "acorn": "bin/acorn"
+      },
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/acorn-jsx": {
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
+      "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
+      "dev": true,
+      "peerDependencies": {
+        "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
+      }
+    },
+    "node_modules/ajv": {
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "dev": true,
+      "dependencies": {
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
+      }
+    },
+    "node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/anymatch": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
+      "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==",
+      "dev": true,
+      "dependencies": {
+        "normalize-path": "^3.0.0",
+        "picomatch": "^2.0.4"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/argparse": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
+      "dev": true
+    },
+    "node_modules/array-buffer-byte-length": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz",
+      "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "is-array-buffer": "^3.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/array-includes": {
+      "version": "3.1.6",
+      "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.6.tgz",
+      "integrity": "sha512-sgTbLvL6cNnw24FnbaDyjmvddQ2ML8arZsgaJhoABMoplz/4QRhtrYS+alr1BUM1Bwp6dhx8vVCBSLG+StwOFw==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.1.4",
+        "es-abstract": "^1.20.4",
+        "get-intrinsic": "^1.1.3",
+        "is-string": "^1.0.7"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/array.prototype.flat": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.1.tgz",
+      "integrity": "sha512-roTU0KWIOmJ4DRLmwKd19Otg0/mT3qPNt0Qb3GWW8iObuZXxrjB/pzn0R3hqpRSWg4HCwqx+0vwOnWnvlOyeIA==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.1.4",
+        "es-abstract": "^1.20.4",
+        "es-shim-unscopables": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/array.prototype.flatmap": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz",
+      "integrity": "sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.1.4",
+        "es-abstract": "^1.20.4",
+        "es-shim-unscopables": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/available-typed-arrays": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
+      "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/balanced-match": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+      "dev": true
+    },
+    "node_modules/binary-extensions": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
+      "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/bootstrap": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-5.2.3.tgz",
+      "integrity": "sha512-cEKPM+fwb3cT8NzQZYEu4HilJ3anCrWqh3CHAok1p9jXqMPsPTBhU25fBckEJHJ/p+tTxTFTsFQGM+gaHpi3QQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/twbs"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/bootstrap"
+        }
+      ],
+      "peerDependencies": {
+        "@popperjs/core": "^2.11.6"
+      }
+    },
+    "node_modules/bootstrap-icons": {
+      "version": "1.10.5",
+      "resolved": "https://registry.npmjs.org/bootstrap-icons/-/bootstrap-icons-1.10.5.tgz",
+      "integrity": "sha512-oSX26F37V7QV7NCE53PPEL45d7EGXmBgHG3pDpZvcRaKVzWMqIRL9wcqJUyEha1esFtM3NJzvmxFXDxjJYD0jQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/twbs"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/bootstrap"
+        }
+      ]
+    },
+    "node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/braces": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "dev": true,
+      "dependencies": {
+        "fill-range": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/builtins": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/builtins/-/builtins-5.0.1.tgz",
+      "integrity": "sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==",
+      "dev": true,
+      "dependencies": {
+        "semver": "^7.0.0"
+      }
+    },
+    "node_modules/builtins/node_modules/semver": {
+      "version": "7.5.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz",
+      "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/call-bind": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
+      "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
+      "dev": true,
+      "dependencies": {
+        "function-bind": "^1.1.1",
+        "get-intrinsic": "^1.0.2"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/callsites": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
+      "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/chokidar": {
+      "version": "3.5.3",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz",
+      "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://paulmillr.com/funding/"
+        }
+      ],
+      "dependencies": {
+        "anymatch": "~3.1.2",
+        "braces": "~3.0.2",
+        "glob-parent": "~5.1.2",
+        "is-binary-path": "~2.1.0",
+        "is-glob": "~4.0.1",
+        "normalize-path": "~3.0.0",
+        "readdirp": "~3.6.0"
+      },
+      "engines": {
+        "node": ">= 8.10.0"
+      },
+      "optionalDependencies": {
+        "fsevents": "~2.3.2"
+      }
+    },
+    "node_modules/chokidar/node_modules/glob-parent": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+      "dev": true,
+      "dependencies": {
+        "is-glob": "^4.0.1"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
+      "dev": true
+    },
+    "node_modules/cross-spawn": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "dev": true,
+      "dependencies": {
+        "path-key": "^3.1.0",
+        "shebang-command": "^2.0.0",
+        "which": "^2.0.1"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/debug": {
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "dev": true,
+      "dependencies": {
+        "ms": "2.1.2"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/deep-is": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
+      "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
+      "dev": true
+    },
+    "node_modules/define-properties": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.0.tgz",
+      "integrity": "sha512-xvqAVKGfT1+UAvPwKTVw/njhdQ8ZhXK4lI0bCIuCMrp2up9nPnaDftrLtmpTazqd1o+UY4zgzU+avtMbDP+ldA==",
+      "dev": true,
+      "dependencies": {
+        "has-property-descriptors": "^1.0.0",
+        "object-keys": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/doctrine": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
+      "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
+      "dev": true,
+      "dependencies": {
+        "esutils": "^2.0.2"
+      },
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
+    "node_modules/es-abstract": {
+      "version": "1.21.2",
+      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.21.2.tgz",
+      "integrity": "sha512-y/B5POM2iBnIxCiernH1G7rC9qQoM77lLIMQLuob0zhp8C56Po81+2Nj0WFKnd0pNReDTnkYryc+zhOzpEIROg==",
+      "dev": true,
+      "dependencies": {
+        "array-buffer-byte-length": "^1.0.0",
+        "available-typed-arrays": "^1.0.5",
+        "call-bind": "^1.0.2",
+        "es-set-tostringtag": "^2.0.1",
+        "es-to-primitive": "^1.2.1",
+        "function.prototype.name": "^1.1.5",
+        "get-intrinsic": "^1.2.0",
+        "get-symbol-description": "^1.0.0",
+        "globalthis": "^1.0.3",
+        "gopd": "^1.0.1",
+        "has": "^1.0.3",
+        "has-property-descriptors": "^1.0.0",
+        "has-proto": "^1.0.1",
+        "has-symbols": "^1.0.3",
+        "internal-slot": "^1.0.5",
+        "is-array-buffer": "^3.0.2",
+        "is-callable": "^1.2.7",
+        "is-negative-zero": "^2.0.2",
+        "is-regex": "^1.1.4",
+        "is-shared-array-buffer": "^1.0.2",
+        "is-string": "^1.0.7",
+        "is-typed-array": "^1.1.10",
+        "is-weakref": "^1.0.2",
+        "object-inspect": "^1.12.3",
+        "object-keys": "^1.1.1",
+        "object.assign": "^4.1.4",
+        "regexp.prototype.flags": "^1.4.3",
+        "safe-regex-test": "^1.0.0",
+        "string.prototype.trim": "^1.2.7",
+        "string.prototype.trimend": "^1.0.6",
+        "string.prototype.trimstart": "^1.0.6",
+        "typed-array-length": "^1.0.4",
+        "unbox-primitive": "^1.0.2",
+        "which-typed-array": "^1.1.9"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/es-set-tostringtag": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz",
+      "integrity": "sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==",
+      "dev": true,
+      "dependencies": {
+        "get-intrinsic": "^1.1.3",
+        "has": "^1.0.3",
+        "has-tostringtag": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-shim-unscopables": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz",
+      "integrity": "sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w==",
+      "dev": true,
+      "dependencies": {
+        "has": "^1.0.3"
+      }
+    },
+    "node_modules/es-to-primitive": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz",
+      "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==",
+      "dev": true,
+      "dependencies": {
+        "is-callable": "^1.1.4",
+        "is-date-object": "^1.0.1",
+        "is-symbol": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/esbuild": {
+      "version": "0.17.18",
+      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.18.tgz",
+      "integrity": "sha512-z1lix43jBs6UKjcZVKOw2xx69ffE2aG0PygLL5qJ9OS/gy0Ewd1gW/PUQIOIQGXBHWNywSc0floSKoMFF8aK2w==",
+      "dev": true,
+      "hasInstallScript": true,
+      "bin": {
+        "esbuild": "bin/esbuild"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "optionalDependencies": {
+        "@esbuild/android-arm": "0.17.18",
+        "@esbuild/android-arm64": "0.17.18",
+        "@esbuild/android-x64": "0.17.18",
+        "@esbuild/darwin-arm64": "0.17.18",
+        "@esbuild/darwin-x64": "0.17.18",
+        "@esbuild/freebsd-arm64": "0.17.18",
+        "@esbuild/freebsd-x64": "0.17.18",
+        "@esbuild/linux-arm": "0.17.18",
+        "@esbuild/linux-arm64": "0.17.18",
+        "@esbuild/linux-ia32": "0.17.18",
+        "@esbuild/linux-loong64": "0.17.18",
+        "@esbuild/linux-mips64el": "0.17.18",
+        "@esbuild/linux-ppc64": "0.17.18",
+        "@esbuild/linux-riscv64": "0.17.18",
+        "@esbuild/linux-s390x": "0.17.18",
+        "@esbuild/linux-x64": "0.17.18",
+        "@esbuild/netbsd-x64": "0.17.18",
+        "@esbuild/openbsd-x64": "0.17.18",
+        "@esbuild/sunos-x64": "0.17.18",
+        "@esbuild/win32-arm64": "0.17.18",
+        "@esbuild/win32-ia32": "0.17.18",
+        "@esbuild/win32-x64": "0.17.18"
+      }
+    },
+    "node_modules/esbuild-sass-plugin": {
+      "version": "2.9.0",
+      "resolved": "https://registry.npmjs.org/esbuild-sass-plugin/-/esbuild-sass-plugin-2.9.0.tgz",
+      "integrity": "sha512-D7c8Ub+C4RfaqhOoo9EEWprYmP5ZCmpmcodmYDtpvCfPZLgsSbLaLHPXdul4TUWUMH5eJ6POw+aes/s42ffwrA==",
+      "dev": true,
+      "dependencies": {
+        "resolve": "^1.22.2",
+        "sass": "^1.62.0"
+      },
+      "peerDependencies": {
+        "esbuild": "^0.17.17"
+      }
+    },
+    "node_modules/escape-string-regexp": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+      "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/eslint": {
+      "version": "8.40.0",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.40.0.tgz",
+      "integrity": "sha512-bvR+TsP9EHL3TqNtj9sCNJVAFK3fBN8Q7g5waghxyRsPLIMwL73XSKnZFK0hk/O2ANC+iAoq6PWMQ+IfBAJIiQ==",
+      "dev": true,
+      "dependencies": {
+        "@eslint-community/eslint-utils": "^4.2.0",
+        "@eslint-community/regexpp": "^4.4.0",
+        "@eslint/eslintrc": "^2.0.3",
+        "@eslint/js": "8.40.0",
+        "@humanwhocodes/config-array": "^0.11.8",
+        "@humanwhocodes/module-importer": "^1.0.1",
+        "@nodelib/fs.walk": "^1.2.8",
+        "ajv": "^6.10.0",
+        "chalk": "^4.0.0",
+        "cross-spawn": "^7.0.2",
+        "debug": "^4.3.2",
+        "doctrine": "^3.0.0",
+        "escape-string-regexp": "^4.0.0",
+        "eslint-scope": "^7.2.0",
+        "eslint-visitor-keys": "^3.4.1",
+        "espree": "^9.5.2",
+        "esquery": "^1.4.2",
+        "esutils": "^2.0.2",
+        "fast-deep-equal": "^3.1.3",
+        "file-entry-cache": "^6.0.1",
+        "find-up": "^5.0.0",
+        "glob-parent": "^6.0.2",
+        "globals": "^13.19.0",
+        "grapheme-splitter": "^1.0.4",
+        "ignore": "^5.2.0",
+        "import-fresh": "^3.0.0",
+        "imurmurhash": "^0.1.4",
+        "is-glob": "^4.0.0",
+        "is-path-inside": "^3.0.3",
+        "js-sdsl": "^4.1.4",
+        "js-yaml": "^4.1.0",
+        "json-stable-stringify-without-jsonify": "^1.0.1",
+        "levn": "^0.4.1",
+        "lodash.merge": "^4.6.2",
+        "minimatch": "^3.1.2",
+        "natural-compare": "^1.4.0",
+        "optionator": "^0.9.1",
+        "strip-ansi": "^6.0.1",
+        "strip-json-comments": "^3.1.0",
+        "text-table": "^0.2.0"
+      },
+      "bin": {
+        "eslint": "bin/eslint.js"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/eslint"
+      }
+    },
+    "node_modules/eslint-config-google": {
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/eslint-config-google/-/eslint-config-google-0.14.0.tgz",
+      "integrity": "sha512-WsbX4WbjuMvTdeVL6+J3rK1RGhCTqjsFjX7UMSMgZiyxxaNLkoJENbrGExzERFeoTpGw3F3FypTiWAP9ZXzkEw==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      },
+      "peerDependencies": {
+        "eslint": ">=5.16.0"
+      }
+    },
+    "node_modules/eslint-config-standard": {
+      "version": "17.0.0",
+      "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-17.0.0.tgz",
+      "integrity": "sha512-/2ks1GKyqSOkH7JFvXJicu0iMpoojkwB+f5Du/1SC0PtBL+s8v30k9njRZ21pm2drKYm2342jFnGWzttxPmZVg==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "peerDependencies": {
+        "eslint": "^8.0.1",
+        "eslint-plugin-import": "^2.25.2",
+        "eslint-plugin-n": "^15.0.0",
+        "eslint-plugin-promise": "^6.0.0"
+      }
+    },
+    "node_modules/eslint-import-resolver-node": {
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.7.tgz",
+      "integrity": "sha512-gozW2blMLJCeFpBwugLTGyvVjNoeo1knonXAcatC6bjPBZitotxdWf7Gimr25N4c0AAOo4eOUfaG82IJPDpqCA==",
+      "dev": true,
+      "dependencies": {
+        "debug": "^3.2.7",
+        "is-core-module": "^2.11.0",
+        "resolve": "^1.22.1"
+      }
+    },
+    "node_modules/eslint-import-resolver-node/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+      "dev": true,
+      "dependencies": {
+        "ms": "^2.1.1"
+      }
+    },
+    "node_modules/eslint-module-utils": {
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.8.0.tgz",
+      "integrity": "sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw==",
+      "dev": true,
+      "dependencies": {
+        "debug": "^3.2.7"
+      },
+      "engines": {
+        "node": ">=4"
+      },
+      "peerDependenciesMeta": {
+        "eslint": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/eslint-module-utils/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+      "dev": true,
+      "dependencies": {
+        "ms": "^2.1.1"
+      }
+    },
+    "node_modules/eslint-plugin-es": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-4.1.0.tgz",
+      "integrity": "sha512-GILhQTnjYE2WorX5Jyi5i4dz5ALWxBIdQECVQavL6s7cI76IZTDWleTHkxz/QT3kvcs2QlGHvKLYsSlPOlPXnQ==",
+      "dev": true,
+      "dependencies": {
+        "eslint-utils": "^2.0.0",
+        "regexpp": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8.10.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/mysticatea"
+      },
+      "peerDependencies": {
+        "eslint": ">=4.19.1"
+      }
+    },
+    "node_modules/eslint-plugin-es/node_modules/eslint-utils": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz",
+      "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==",
+      "dev": true,
+      "dependencies": {
+        "eslint-visitor-keys": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/mysticatea"
+      }
+    },
+    "node_modules/eslint-plugin-es/node_modules/eslint-visitor-keys": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+      "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/eslint-plugin-import": {
+      "version": "2.27.5",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.27.5.tgz",
+      "integrity": "sha512-LmEt3GVofgiGuiE+ORpnvP+kAm3h6MLZJ4Q5HCyHADofsb4VzXFsRiWj3c0OFiV+3DWFh0qg3v9gcPlfc3zRow==",
+      "dev": true,
+      "dependencies": {
+        "array-includes": "^3.1.6",
+        "array.prototype.flat": "^1.3.1",
+        "array.prototype.flatmap": "^1.3.1",
+        "debug": "^3.2.7",
+        "doctrine": "^2.1.0",
+        "eslint-import-resolver-node": "^0.3.7",
+        "eslint-module-utils": "^2.7.4",
+        "has": "^1.0.3",
+        "is-core-module": "^2.11.0",
+        "is-glob": "^4.0.3",
+        "minimatch": "^3.1.2",
+        "object.values": "^1.1.6",
+        "resolve": "^1.22.1",
+        "semver": "^6.3.0",
+        "tsconfig-paths": "^3.14.1"
+      },
+      "engines": {
+        "node": ">=4"
+      },
+      "peerDependencies": {
+        "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8"
+      }
+    },
+    "node_modules/eslint-plugin-import/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+      "dev": true,
+      "dependencies": {
+        "ms": "^2.1.1"
+      }
+    },
+    "node_modules/eslint-plugin-import/node_modules/doctrine": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
+      "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==",
+      "dev": true,
+      "dependencies": {
+        "esutils": "^2.0.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/eslint-plugin-n": {
+      "version": "15.7.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-n/-/eslint-plugin-n-15.7.0.tgz",
+      "integrity": "sha512-jDex9s7D/Qial8AGVIHq4W7NswpUD5DPDL2RH8Lzd9EloWUuvUkHfv4FRLMipH5q2UtyurorBkPeNi1wVWNh3Q==",
+      "dev": true,
+      "dependencies": {
+        "builtins": "^5.0.1",
+        "eslint-plugin-es": "^4.1.0",
+        "eslint-utils": "^3.0.0",
+        "ignore": "^5.1.1",
+        "is-core-module": "^2.11.0",
+        "minimatch": "^3.1.2",
+        "resolve": "^1.22.1",
+        "semver": "^7.3.8"
+      },
+      "engines": {
+        "node": ">=12.22.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/mysticatea"
+      },
+      "peerDependencies": {
+        "eslint": ">=7.0.0"
+      }
+    },
+    "node_modules/eslint-plugin-n/node_modules/semver": {
+      "version": "7.5.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz",
+      "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/eslint-plugin-promise": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-6.1.1.tgz",
+      "integrity": "sha512-tjqWDwVZQo7UIPMeDReOpUgHCmCiH+ePnVT+5zVapL0uuHnegBUs2smM13CzOs2Xb5+MHMRFTs9v24yjba4Oig==",
+      "dev": true,
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "peerDependencies": {
+        "eslint": "^7.0.0 || ^8.0.0"
+      }
+    },
+    "node_modules/eslint-scope": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.0.tgz",
+      "integrity": "sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==",
+      "dev": true,
+      "dependencies": {
+        "esrecurse": "^4.3.0",
+        "estraverse": "^5.2.0"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/eslint"
+      }
+    },
+    "node_modules/eslint-utils": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz",
+      "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==",
+      "dev": true,
+      "dependencies": {
+        "eslint-visitor-keys": "^2.0.0"
+      },
+      "engines": {
+        "node": "^10.0.0 || ^12.0.0 || >= 14.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/mysticatea"
+      },
+      "peerDependencies": {
+        "eslint": ">=5"
+      }
+    },
+    "node_modules/eslint-utils/node_modules/eslint-visitor-keys": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz",
+      "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/eslint-visitor-keys": {
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz",
+      "integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==",
+      "dev": true,
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/eslint"
+      }
+    },
+    "node_modules/espree": {
+      "version": "9.5.2",
+      "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.2.tgz",
+      "integrity": "sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw==",
+      "dev": true,
+      "dependencies": {
+        "acorn": "^8.8.0",
+        "acorn-jsx": "^5.3.2",
+        "eslint-visitor-keys": "^3.4.1"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/eslint"
+      }
+    },
+    "node_modules/esquery": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz",
+      "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==",
+      "dev": true,
+      "dependencies": {
+        "estraverse": "^5.1.0"
+      },
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
+    "node_modules/esrecurse": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+      "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+      "dev": true,
+      "dependencies": {
+        "estraverse": "^5.2.0"
+      },
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
+    "node_modules/estraverse": {
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
+      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+      "dev": true,
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
+    "node_modules/esutils": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
+      "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/event-source-polyfill": {
+      "version": "1.0.31",
+      "resolved": "https://registry.npmjs.org/event-source-polyfill/-/event-source-polyfill-1.0.31.tgz",
+      "integrity": "sha512-4IJSItgS/41IxN5UVAVuAyczwZF7ZIEsM1XAoUzIHA6A+xzusEZUutdXz2Nr+MQPLxfTiCvqE79/C8HT8fKFvA=="
+    },
+    "node_modules/fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "dev": true
+    },
+    "node_modules/fast-json-stable-stringify": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+      "dev": true
+    },
+    "node_modules/fast-levenshtein": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
+      "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
+      "dev": true
+    },
+    "node_modules/fastq": {
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz",
+      "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==",
+      "dev": true,
+      "dependencies": {
+        "reusify": "^1.0.4"
+      }
+    },
+    "node_modules/file-entry-cache": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
+      "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
+      "dev": true,
+      "dependencies": {
+        "flat-cache": "^3.0.4"
+      },
+      "engines": {
+        "node": "^10.12.0 || >=12.0.0"
+      }
+    },
+    "node_modules/fill-range": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "dev": true,
+      "dependencies": {
+        "to-regex-range": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/find-up": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
+      "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==",
+      "dev": true,
+      "dependencies": {
+        "locate-path": "^6.0.0",
+        "path-exists": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/flat-cache": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz",
+      "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==",
+      "dev": true,
+      "dependencies": {
+        "flatted": "^3.1.0",
+        "rimraf": "^3.0.2"
+      },
+      "engines": {
+        "node": "^10.12.0 || >=12.0.0"
+      }
+    },
+    "node_modules/flatted": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz",
+      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
+      "dev": true
+    },
+    "node_modules/for-each": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
+      "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==",
+      "dev": true,
+      "dependencies": {
+        "is-callable": "^1.1.3"
+      }
+    },
+    "node_modules/fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
+      "dev": true
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/function-bind": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
+      "dev": true
+    },
+    "node_modules/function.prototype.name": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.5.tgz",
+      "integrity": "sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.1.3",
+        "es-abstract": "^1.19.0",
+        "functions-have-names": "^1.2.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/functions-have-names": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz",
+      "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==",
+      "dev": true,
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/get-intrinsic": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz",
+      "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==",
+      "dev": true,
+      "dependencies": {
+        "function-bind": "^1.1.1",
+        "has": "^1.0.3",
+        "has-symbols": "^1.0.3"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/get-symbol-description": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz",
+      "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "get-intrinsic": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/glob-parent": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
+      "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+      "dev": true,
+      "dependencies": {
+        "is-glob": "^4.0.3"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    },
+    "node_modules/globals": {
+      "version": "13.20.0",
+      "resolved": "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz",
+      "integrity": "sha512-Qg5QtVkCy/kv3FUSlu4ukeZDVf9ee0iXLAUYX13gbR17bnejFTzr4iS9bY7kwCf1NztRNm1t91fjOiyx4CSwPQ==",
+      "dev": true,
+      "dependencies": {
+        "type-fest": "^0.20.2"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/globalthis": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz",
+      "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==",
+      "dev": true,
+      "dependencies": {
+        "define-properties": "^1.1.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/gopd": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+      "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+      "dev": true,
+      "dependencies": {
+        "get-intrinsic": "^1.1.3"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/grapheme-splitter": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz",
+      "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==",
+      "dev": true
+    },
+    "node_modules/has": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
+      "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
+      "dev": true,
+      "dependencies": {
+        "function-bind": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4.0"
+      }
+    },
+    "node_modules/has-bigints": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz",
+      "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==",
+      "dev": true,
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/has-property-descriptors": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz",
+      "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==",
+      "dev": true,
+      "dependencies": {
+        "get-intrinsic": "^1.1.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
+      "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-symbols": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
+      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-tostringtag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
+      "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
+      "dev": true,
+      "dependencies": {
+        "has-symbols": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/ignore": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz",
+      "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==",
+      "dev": true,
+      "engines": {
+        "node": ">= 4"
+      }
+    },
+    "node_modules/immutable": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.0.tgz",
+      "integrity": "sha512-0AOCmOip+xgJwEVTQj1EfiDDOkPmuyllDuTuEX+DDXUgapLAsBIfkg3sxCYyCEA8mQqZrrxPUGjcOQ2JS3WLkg==",
+      "dev": true
+    },
+    "node_modules/import-fresh": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
+      "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+      "dev": true,
+      "dependencies": {
+        "parent-module": "^1.0.0",
+        "resolve-from": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/imurmurhash": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
+      "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.8.19"
+      }
+    },
+    "node_modules/inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+      "dev": true,
+      "dependencies": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
+    "node_modules/inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "dev": true
+    },
+    "node_modules/internal-slot": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz",
+      "integrity": "sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ==",
+      "dev": true,
+      "dependencies": {
+        "get-intrinsic": "^1.2.0",
+        "has": "^1.0.3",
+        "side-channel": "^1.0.4"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/is-array-buffer": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz",
+      "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "get-intrinsic": "^1.2.0",
+        "is-typed-array": "^1.1.10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-bigint": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz",
+      "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==",
+      "dev": true,
+      "dependencies": {
+        "has-bigints": "^1.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-binary-path": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
+      "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
+      "dev": true,
+      "dependencies": {
+        "binary-extensions": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/is-boolean-object": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz",
+      "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "has-tostringtag": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-callable": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz",
+      "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-core-module": {
+      "version": "2.12.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz",
+      "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==",
+      "dev": true,
+      "dependencies": {
+        "has": "^1.0.3"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-date-object": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz",
+      "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==",
+      "dev": true,
+      "dependencies": {
+        "has-tostringtag": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-extglob": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+      "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/is-glob": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
+      "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
+      "dev": true,
+      "dependencies": {
+        "is-extglob": "^2.1.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/is-negative-zero": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz",
+      "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-number": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+      "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.12.0"
+      }
+    },
+    "node_modules/is-number-object": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz",
+      "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==",
+      "dev": true,
+      "dependencies": {
+        "has-tostringtag": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-path-inside": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz",
+      "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/is-regex": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz",
+      "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "has-tostringtag": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-shared-array-buffer": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz",
+      "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-string": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz",
+      "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==",
+      "dev": true,
+      "dependencies": {
+        "has-tostringtag": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-symbol": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz",
+      "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==",
+      "dev": true,
+      "dependencies": {
+        "has-symbols": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-typed-array": {
+      "version": "1.1.10",
+      "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.10.tgz",
+      "integrity": "sha512-PJqgEHiWZvMpaFZ3uTc8kHPM4+4ADTlDniuQL7cU/UDA0Ql7F70yGfHph3cLNe+c9toaigv+DFzTJKhc2CtO6A==",
+      "dev": true,
+      "dependencies": {
+        "available-typed-arrays": "^1.0.5",
+        "call-bind": "^1.0.2",
+        "for-each": "^0.3.3",
+        "gopd": "^1.0.1",
+        "has-tostringtag": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-weakref": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz",
+      "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
+      "dev": true
+    },
+    "node_modules/js-sdsl": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.0.tgz",
+      "integrity": "sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg==",
+      "dev": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/js-sdsl"
+      }
+    },
+    "node_modules/js-yaml": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+      "dev": true,
+      "dependencies": {
+        "argparse": "^2.0.1"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
+    "node_modules/json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+      "dev": true
+    },
+    "node_modules/json-stable-stringify-without-jsonify": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
+      "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==",
+      "dev": true
+    },
+    "node_modules/json5": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz",
+      "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==",
+      "dev": true,
+      "dependencies": {
+        "minimist": "^1.2.0"
+      },
+      "bin": {
+        "json5": "lib/cli.js"
+      }
+    },
+    "node_modules/jsonpath-plus": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz",
+      "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==",
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
+    "node_modules/levn": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
+      "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
+      "dev": true,
+      "dependencies": {
+        "prelude-ls": "^1.2.1",
+        "type-check": "~0.4.0"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/locate-path": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
+      "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==",
+      "dev": true,
+      "dependencies": {
+        "p-locate": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
+    "node_modules/lodash.merge": {
+      "version": "4.6.2",
+      "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
+      "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
+      "dev": true
+    },
+    "node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/minimist": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+      "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+      "dev": true,
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/ms": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+      "dev": true
+    },
+    "node_modules/natural-compare": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
+      "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
+      "dev": true
+    },
+    "node_modules/normalize-path": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
+      "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/object-inspect": {
+      "version": "1.12.3",
+      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz",
+      "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==",
+      "dev": true,
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/object-keys": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
+      "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/object.assign": {
+      "version": "4.1.4",
+      "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz",
+      "integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.1.4",
+        "has-symbols": "^1.0.3",
+        "object-keys": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/object.values": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.6.tgz",
+      "integrity": "sha512-FVVTkD1vENCsAcwNs9k6jea2uHC/X0+JcjG8YA60FN5CMaJmG95wT9jek/xX9nornqGRrBkKtzuAu2wuHpKqvw==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.1.4",
+        "es-abstract": "^1.20.4"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "dev": true,
+      "dependencies": {
+        "wrappy": "1"
+      }
+    },
+    "node_modules/optionator": {
+      "version": "0.9.1",
+      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
+      "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
+      "dev": true,
+      "dependencies": {
+        "deep-is": "^0.1.3",
+        "fast-levenshtein": "^2.0.6",
+        "levn": "^0.4.1",
+        "prelude-ls": "^1.2.1",
+        "type-check": "^0.4.0",
+        "word-wrap": "^1.2.3"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/p-limit": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+      "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
+      "dev": true,
+      "dependencies": {
+        "yocto-queue": "^0.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-locate": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
+      "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
+      "dev": true,
+      "dependencies": {
+        "p-limit": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/parent-module": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
+      "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
+      "dev": true,
+      "dependencies": {
+        "callsites": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/path-exists": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+      "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/path-key": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/path-parse": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
+      "dev": true
+    },
+    "node_modules/picomatch": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
+      "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
+      "dev": true,
+      "engines": {
+        "node": ">=8.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/jonschlinkert"
+      }
+    },
+    "node_modules/prelude-ls": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
+      "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/punycode": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
+      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/queue-microtask": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
+      "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
+    "node_modules/readdirp": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
+      "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==",
+      "dev": true,
+      "dependencies": {
+        "picomatch": "^2.2.1"
+      },
+      "engines": {
+        "node": ">=8.10.0"
+      }
+    },
+    "node_modules/regexp.prototype.flags": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz",
+      "integrity": "sha512-0SutC3pNudRKgquxGoRGIz946MZVHqbNfPjBdxeOhBrdgDKlRoXmYLQN9xRbrR09ZXWeGAdPuif7egofn6v5LA==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.2.0",
+        "functions-have-names": "^1.2.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/regexpp": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz",
+      "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/mysticatea"
+      }
+    },
+    "node_modules/resolve": {
+      "version": "1.22.2",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz",
+      "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==",
+      "dev": true,
+      "dependencies": {
+        "is-core-module": "^2.11.0",
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
+      },
+      "bin": {
+        "resolve": "bin/resolve"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/resolve-from": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
+      "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/reusify": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
+      "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+      "dev": true,
+      "engines": {
+        "iojs": ">=1.0.0",
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/rimraf": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "dev": true,
+      "dependencies": {
+        "glob": "^7.1.3"
+      },
+      "bin": {
+        "rimraf": "bin.js"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/run-parallel": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
+      "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "queue-microtask": "^1.2.2"
+      }
+    },
+    "node_modules/safe-regex-test": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz",
+      "integrity": "sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "get-intrinsic": "^1.1.3",
+        "is-regex": "^1.1.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/sass": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/sass/-/sass-1.62.1.tgz",
+      "integrity": "sha512-NHpxIzN29MXvWiuswfc1W3I0N8SXBd8UR26WntmDlRYf0bSADnwnOjsyMZ3lMezSlArD33Vs3YFhp7dWvL770A==",
+      "dev": true,
+      "dependencies": {
+        "chokidar": ">=3.0.0 <4.0.0",
+        "immutable": "^4.0.0",
+        "source-map-js": ">=0.6.2 <2.0.0"
+      },
+      "bin": {
+        "sass": "sass.js"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/semver": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+      "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+      "dev": true,
+      "bin": {
+        "semver": "bin/semver.js"
+      }
+    },
+    "node_modules/shebang-command": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+      "dev": true,
+      "dependencies": {
+        "shebang-regex": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/shebang-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/side-channel": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
+      "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.0",
+        "get-intrinsic": "^1.0.2",
+        "object-inspect": "^1.9.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/source-map-js": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz",
+      "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/string.prototype.trim": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz",
+      "integrity": "sha512-p6TmeT1T3411M8Cgg9wBTMRtY2q9+PNy9EV1i2lIXUN/btt763oIfxwN3RR8VU6wHX8j/1CFy0L+YuThm6bgOg==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.1.4",
+        "es-abstract": "^1.20.4"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/string.prototype.trimend": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz",
+      "integrity": "sha512-JySq+4mrPf9EsDBEDYMOb/lM7XQLulwg5R/m1r0PXEFqrV0qHvl58sdTilSXtKOflCsK2E8jxf+GKC0T07RWwQ==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.1.4",
+        "es-abstract": "^1.20.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/string.prototype.trimstart": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz",
+      "integrity": "sha512-omqjMDaY92pbn5HOX7f9IccLA+U1tA9GvtU4JrodiXFfYB7jPzzHpRzpglLAjtUV6bB557zwClJezTqnAiYnQA==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.1.4",
+        "es-abstract": "^1.20.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dev": true,
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-bom": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+      "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/strip-json-comments": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+      "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/supports-preserve-symlinks-flag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
+      "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/text-table": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
+      "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
+      "dev": true
+    },
+    "node_modules/to-regex-range": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+      "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+      "dev": true,
+      "dependencies": {
+        "is-number": "^7.0.0"
+      },
+      "engines": {
+        "node": ">=8.0"
+      }
+    },
+    "node_modules/tsconfig-paths": {
+      "version": "3.14.2",
+      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz",
+      "integrity": "sha512-o/9iXgCYc5L/JxCHPe3Hvh8Q/2xm5Z+p18PESBU6Ff33695QnCHBEjcytY2q19ua7Mbl/DavtBOLq+oG0RCL+g==",
+      "dev": true,
+      "dependencies": {
+        "@types/json5": "^0.0.29",
+        "json5": "^1.0.2",
+        "minimist": "^1.2.6",
+        "strip-bom": "^3.0.0"
+      }
+    },
+    "node_modules/type-check": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
+      "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
+      "dev": true,
+      "dependencies": {
+        "prelude-ls": "^1.2.1"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/type-fest": {
+      "version": "0.20.2",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/typed-array-length": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.4.tgz",
+      "integrity": "sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "for-each": "^0.3.3",
+        "is-typed-array": "^1.1.9"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/unbox-primitive": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz",
+      "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "has-bigints": "^1.0.2",
+        "has-symbols": "^1.0.3",
+        "which-boxed-primitive": "^1.0.2"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/uri-js": {
+      "version": "4.4.1",
+      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+      "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+      "dev": true,
+      "dependencies": {
+        "punycode": "^2.1.0"
+      }
+    },
+    "node_modules/which": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+      "dev": true,
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/node-which"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/which-boxed-primitive": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz",
+      "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==",
+      "dev": true,
+      "dependencies": {
+        "is-bigint": "^1.0.1",
+        "is-boolean-object": "^1.1.0",
+        "is-number-object": "^1.0.4",
+        "is-string": "^1.0.5",
+        "is-symbol": "^1.0.3"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/which-typed-array": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.9.tgz",
+      "integrity": "sha512-w9c4xkx6mPidwp7180ckYWfMmvxpjlZuIudNtDf4N/tTAUB8VJbX25qZoAsrtGuYNnGw3pa0AXgbGKRB8/EceA==",
+      "dev": true,
+      "dependencies": {
+        "available-typed-arrays": "^1.0.5",
+        "call-bind": "^1.0.2",
+        "for-each": "^0.3.3",
+        "gopd": "^1.0.1",
+        "has-tostringtag": "^1.0.0",
+        "is-typed-array": "^1.1.10"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/word-wrap": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
+      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+      "dev": true
+    },
+    "node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/yocto-queue": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
+      "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    }
+  }
+}
diff --git a/ui/package.json b/ui/package.json
new file mode 100644
index 00000000000..496619fc1ce
--- /dev/null
+++ b/ui/package.json
@@ -0,0 +1,31 @@
+{
+  "name": "ui",
+  "version": "1.0.0",
+  "description": "",
+  "private": true,
+  "scripts": {
+    "start": "node build.mjs serve",
+    "build": "node build.mjs"
+  },
+  "author": "",
+  "license": "EPL-2.0",
+  "devDependencies": {
+    "esbuild": "^0.17.18",
+    "esbuild-sass-plugin": "^2.9.0",
+    "eslint": "^8.40.0",
+    "eslint-config-google": "^0.14.0",
+    "eslint-config-standard": "^17.0.0",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-promise": "^6.1.1"
+  },
+  "dependencies": {
+    "@popperjs/core": "^2.11.7",
+    "@tarekraafat/autocomplete.js": "^10.2.7",
+    "bootstrap": "^5.2.3",
+    "bootstrap-icons": "^1.10.5",
+    "event-source-polyfill": "^1.0.31",
+    "jsonpath-plus": "^7.2.0",
+    "lodash": "^4.17.21"
+  }
+}
diff --git a/ui/readme.md b/ui/readme.md
new file mode 100644
index 00000000000..943d8447629
--- /dev/null
+++ b/ui/readme.md
@@ -0,0 +1,15 @@
+# Getting started
+
+## Development
+
+To start development server use
+
+`npm run start`
+
+Browse to the UI
+
+`http://localhost:8000/`
+
+## Build for production
+
+`npm run build`
\ No newline at end of file
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/config/DefaultToThingDescriptionConfig.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/config/DefaultToThingDescriptionConfig.java
index 4520a8e5cda..172775dbd36 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/config/DefaultToThingDescriptionConfig.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/config/DefaultToThingDescriptionConfig.java
@@ -44,9 +44,13 @@ final class DefaultToThingDescriptionConfig implements ToThingDescriptionConfig
 
     private DefaultToThingDescriptionConfig(final ScopedConfig scopedConfig) {
         basePrefix = scopedConfig.getString(ConfigValue.BASE_PREFIX.getConfigPath());
-        jsonTemplate = JsonFactory.readFrom(
-                scopedConfig.getValue(ConfigValue.JSON_TEMPLATE.getConfigPath()).render(ConfigRenderOptions.concise())
-        ).asObject();
+        final String jsonTemplateRendered =
+                scopedConfig.getValue(ConfigValue.JSON_TEMPLATE.getConfigPath()).render(ConfigRenderOptions.concise());
+        final String jsonTemplateStrippedStartEndQuotes =
+                (jsonTemplateRendered.startsWith("\"{") && jsonTemplateRendered.endsWith("}\"")) ?
+                jsonTemplateRendered.substring(1, jsonTemplateRendered.length()-1) : jsonTemplateRendered;
+        final String replaceEscapedQuotesJsonTemplate = jsonTemplateStrippedStartEndQuotes.replace("\\\"", "\"");
+        jsonTemplate = JsonFactory.readFrom(replaceEscapedQuotesJsonTemplate).asObject();
         placeholders = JsonFactory.readFrom(
                 scopedConfig.getValue(ConfigValue.PLACEHOLDERS.getConfigPath()).render(ConfigRenderOptions.concise())
         ).asObject()
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingDescriptionGenerator.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingDescriptionGenerator.java
index f084c3cd577..9bc61a95fbb 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingDescriptionGenerator.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingDescriptionGenerator.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CompletionStage;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 import java.util.regex.Matcher;
@@ -135,7 +136,7 @@ final class DefaultWotThingDescriptionGenerator implements WotThingDescriptionGe
     }
 
     @Override
-    public ThingDescription generateThingDescription(final ThingId thingId,
+    public CompletionStage generateThingDescription(final ThingId thingId,
             @Nullable final Thing thing,
             @Nullable final JsonObject placeholderLookupObject,
             @Nullable final String featureId,
@@ -144,46 +145,51 @@ public ThingDescription generateThingDescription(final ThingId thingId,
             final DittoHeaders dittoHeaders) {
 
         // generation rules defined at: https://w3c.github.io/wot-thing-description/#thing-model-td-generation
+        return thingModelExtensionResolver
+                .resolveThingModelExtensions(thingModel, dittoHeaders)
+                .thenCompose(thingModelWithExtensions ->
+                        thingModelExtensionResolver.resolveThingModelRefs(thingModelWithExtensions, dittoHeaders)
+                )
+                .thenApply(thingModelWithExtensionsAndImports -> {
+                    LOGGER.withCorrelationId(dittoHeaders)
+                            .debug("ThingModel after resolving extensions + refs: <{}>",
+                                    thingModelWithExtensionsAndImports);
+
+                    final ThingModel cleanedTm =
+                            removeThingModelSpecificElements(thingModelWithExtensionsAndImports, dittoHeaders);
+
+                    final ThingDescription.Builder tdBuilder = ThingDescription.newBuilder(cleanedTm.toJson());
+                    addBase(tdBuilder, thingId, featureId);
+                    addInstanceVersion(tdBuilder, cleanedTm.getVersion().orElse(null));
+                    addThingDescriptionLinks(tdBuilder, thingModelUrl, null != featureId, thingId);
+                    convertThingDescriptionTmSubmodelLinksToItems(tdBuilder, dittoHeaders);
+                    addThingDescriptionTemplateFromConfig(tdBuilder);
+                    addThingDescriptionAdditionalMetadata(tdBuilder, thing);
+                    if (null == featureId) {
+                        addThingDescriptionForms(cleanedTm, tdBuilder, Thing.JsonFields.ATTRIBUTES.getPointer());
+                    } else {
+                        addThingDescriptionForms(cleanedTm, tdBuilder, Feature.JsonFields.PROPERTIES.getPointer());
+                    }
 
-        final ThingModel thingModelWithExtensions = thingModelExtensionResolver
-                .resolveThingModelExtensions(thingModel, dittoHeaders);
-        final ThingModel thingModelWithExtensionsAndImports = thingModelExtensionResolver
-                .resolveThingModelRefs(thingModelWithExtensions, dittoHeaders);
-
-        LOGGER.withCorrelationId(dittoHeaders)
-                .debug("ThingModel after resolving extensions + refs: <{}>", thingModelWithExtensionsAndImports);
-
-        final ThingModel cleanedTm = removeThingModelSpecificElements(thingModelWithExtensionsAndImports, dittoHeaders);
-
-        final ThingDescription.Builder tdBuilder = ThingDescription.newBuilder(cleanedTm.toJson());
-        addBase(tdBuilder, thingId, featureId);
-        addInstanceVersion(tdBuilder, cleanedTm.getVersion().orElse(null));
-        addThingDescriptionLinks(tdBuilder, thingModelUrl, null != featureId, thingId);
-        convertThingDescriptionTmSubmodelLinksToItems(tdBuilder, dittoHeaders);
-        addThingDescriptionTemplateFromConfig(tdBuilder);
-        addThingDescriptionAdditionalMetadata(tdBuilder, thing);
-        if (null == featureId) {
-            addThingDescriptionForms(cleanedTm, tdBuilder, Thing.JsonFields.ATTRIBUTES.getPointer());
-        } else {
-            addThingDescriptionForms(cleanedTm, tdBuilder, Feature.JsonFields.PROPERTIES.getPointer());
-        }
-
-        tdBuilder.setUriVariables(provideUriVariables(
-                DittoHeaderDefinition.CHANNEL.getKey(),
-                DittoHeaderDefinition.TIMEOUT.getKey(),
-                DittoHeaderDefinition.RESPONSE_REQUIRED.getKey(),
-                DITTO_FIELDS_URI_VARIABLE
-        ));
-        tdBuilder.setSchemaDefinitions(SchemaDefinitions.of(
-                Map.of(SCHEMA_DITTO_ERROR, buildDittoErrorSchema())
-        ));
-
-        final ThingDescription thingDescription = resolvePlaceholders(tdBuilder.build(), placeholderLookupObject,
-                dittoHeaders);
-        LOGGER.withCorrelationId(dittoHeaders)
-                .info("Created ThingDescription for thingId <{}> and featureId <{}>: <{}>", thingId, featureId,
-                        thingDescription);
-        return thingDescription;
+                    tdBuilder.setUriVariables(provideUriVariables(
+                            DittoHeaderDefinition.CHANNEL.getKey(),
+                            DittoHeaderDefinition.TIMEOUT.getKey(),
+                            DittoHeaderDefinition.RESPONSE_REQUIRED.getKey(),
+                            DITTO_FIELDS_URI_VARIABLE
+                    ));
+                    tdBuilder.setSchemaDefinitions(SchemaDefinitions.of(
+                            Map.of(SCHEMA_DITTO_ERROR, buildDittoErrorSchema())
+                    ));
+
+                    final ThingDescription thingDescription =
+                            resolvePlaceholders(tdBuilder.build(), placeholderLookupObject,
+                                    dittoHeaders);
+                    LOGGER.withCorrelationId(dittoHeaders)
+                            .info("Created ThingDescription for thingId <{}> and featureId <{}>: <{}>", thingId,
+                                    featureId,
+                                    thingDescription);
+                    return thingDescription;
+                });
     }
 
     private ObjectSchema buildDittoErrorSchema() {
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingModelExtensionResolver.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingModelExtensionResolver.java
index 6763c0b2b8a..6a52e95765d 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingModelExtensionResolver.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingModelExtensionResolver.java
@@ -18,6 +18,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Executor;
+import java.util.function.BiFunction;
 
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.json.JsonCollectors;
@@ -49,58 +50,85 @@ final class DefaultWotThingModelExtensionResolver implements WotThingModelExtens
     }
 
     @Override
-    public ThingModel resolveThingModelExtensions(final ThingModel thingModel, final DittoHeaders dittoHeaders) {
+    public CompletionStage resolveThingModelExtensions(final ThingModel thingModel,
+            final DittoHeaders dittoHeaders) {
         final ThingModel.Builder tmBuilder = thingModel.toBuilder();
-        thingModel.getLinks()
+        return thingModel.getLinks()
                 .map(links -> {
                             final List> fetchedModelFutures = links.stream()
                                     .filter(baseLink -> baseLink.getRel().filter(TM_EXTENDS::equals).isPresent())
                                     .map(extendsLink -> thingModelFetcher.fetchThingModel(extendsLink.getHref(), dittoHeaders))
                                     .map(CompletionStage::toCompletableFuture)
                                     .toList();
-                            final CompletableFuture allModelFuture =
+                            final CompletionStage allModelFuture =
                                     CompletableFuture.allOf(fetchedModelFutures.toArray(new CompletableFuture[0]));
-                            return allModelFuture.thenApplyAsync(aVoid -> fetchedModelFutures.stream()
+                            return allModelFuture
+                                    .thenApplyAsync(aVoid -> fetchedModelFutures.stream()
                                             .map(CompletableFuture::join) // joining does not block anything here as "allOf" already guaranteed that all futures are ready
-                                            .toList(), executor)
-                                    .join();
+                                            .toList(), executor
+                                    );
                         }
                 )
-                .ifPresent(extendedModels -> extendedModels.forEach(extendedModel -> {
-                    final ThingModel extendedRecursedModel =
-                            resolveThingModelExtensions(extendedModel, dittoHeaders); // recurse!
+                .map(extendedModelsFut -> extendedModelsFut.thenComposeAsync(extendedModels -> {
+                    if (extendedModels.isEmpty()) {
+                        return CompletableFuture.completedStage(thingModel);
+                    } else {
+                        CompletionStage currentStage =
+                                resolveThingModelExtensions(extendedModels.get(0), dittoHeaders) // recurse!
+                                        .thenApply(extendedModel ->
+                                                mergeThingModelIntoBuilder().apply(tmBuilder, extendedModel)
+                                        );
+                        for (int i = 1; i < extendedModels.size(); i++) {
+                            currentStage = currentStage.thenCombine(
+                                    resolveThingModelExtensions(extendedModels.get(i), dittoHeaders),  // recurse!
+                                    mergeThingModelIntoBuilder()
+                            );
+                        }
+                        return currentStage.thenApply(ThingModel.Builder::build);
+                    }
+                }, executor))
+                .orElse(CompletableFuture.completedStage(thingModel));
+    }
 
-                    final JsonObject mergedTmObject = JsonFactory
-                            .mergeJsonValues(tmBuilder.build(), extendedRecursedModel).asObject();
-                    tmBuilder.removeAll();
-                    tmBuilder.setAll(mergedTmObject);
-                }));
-        return tmBuilder.build();
+    private BiFunction mergeThingModelIntoBuilder() {
+        return (builder, model) -> {
+            final JsonObject mergedTmObject = JsonFactory.mergeJsonValues(builder.build(), model).asObject();
+            builder.removeAll();
+            builder.setAll(mergedTmObject);
+            return builder;
+        };
     }
 
     @Override
-    public ThingModel resolveThingModelRefs(final ThingModel thingModel, final DittoHeaders dittoHeaders) {
-        final JsonObject thingModelObject = potentiallyResolveRefs(thingModel, dittoHeaders);
-        return ThingModel.fromJson(thingModelObject);
+    public CompletionStage resolveThingModelRefs(final ThingModel thingModel, final DittoHeaders dittoHeaders) {
+        return potentiallyResolveRefs(thingModel, dittoHeaders).thenApply(ThingModel::fromJson);
     }
 
-    private JsonObject potentiallyResolveRefs(final JsonObject jsonObject, final DittoHeaders dittoHeaders) {
-        return jsonObject.stream()
+    private CompletionStage potentiallyResolveRefs(final JsonObject jsonObject,
+            final DittoHeaders dittoHeaders) {
+        final List> completionStages = jsonObject.stream()
                 .map(field -> {
                     if (field.getValue().isObject() && field.getValue().asObject().contains(TM_REF)) {
-                        return JsonField.newInstance(field.getKey(),
-                                resolveRefs(field.getValue().asObject(), dittoHeaders));
+                        return resolveRefs(field.getValue().asObject(), dittoHeaders)
+                                .thenApply(refs -> JsonField.newInstance(field.getKey(), refs));
                     } else if (field.getValue().isObject()) {
-                        return JsonField.newInstance(field.getKey(),
-                                potentiallyResolveRefs(field.getValue().asObject(), dittoHeaders)); // recurse!
+                        return potentiallyResolveRefs(field.getValue().asObject(), dittoHeaders)  // recurse!
+                                .thenApply(refs -> JsonField.newInstance(field.getKey(), refs));
                     } else {
-                        return field;
+                        return CompletableFuture.completedStage(field);
                     }
                 })
-                .collect(JsonCollectors.fieldsToObject());
+                .map(CompletionStage::toCompletableFuture)
+                .toList();
+        return CompletableFuture.allOf(completionStages.toArray(CompletableFuture[]::new))
+                .thenApplyAsync(v -> completionStages.stream()
+                                .map(CompletableFuture::join)
+                                .collect(JsonCollectors.fieldsToObject()),
+                        executor
+                );
     }
 
-    private JsonValue resolveRefs(final JsonObject objectWithTmRef, final DittoHeaders dittoHeaders) {
+    private CompletionStage resolveRefs(final JsonObject objectWithTmRef, final DittoHeaders dittoHeaders) {
         final String tmRef = objectWithTmRef.getValue(TM_REF)
                 .filter(JsonValue::isString)
                 .map(JsonValue::asString)
@@ -110,15 +138,17 @@ private JsonValue resolveRefs(final JsonObject objectWithTmRef, final DittoHeade
         if (urlAndPointer.length != 2) {
             throw WotThingModelRefInvalidException.newBuilder(tmRef).dittoHeaders(dittoHeaders).build();
         }
-        final JsonObject refObject = thingModelFetcher.fetchThingModel(IRI.of(urlAndPointer[0]), dittoHeaders)
+        return thingModelFetcher.fetchThingModel(IRI.of(urlAndPointer[0]), dittoHeaders)
                 .thenApply(thingModel -> thingModel.getValue(JsonPointer.of(urlAndPointer[1])))
                 .thenComposeAsync(optJsonValue -> optJsonValue
                                 .filter(JsonValue::isObject)
                                 .map(JsonValue::asObject)
                                 .map(CompletableFuture::completedStage)
                                 .orElseGet(() -> CompletableFuture.completedStage(null))
-                        , executor).toCompletableFuture().join();
-
-        return JsonFactory.mergeJsonValues(objectWithTmRef.remove(TM_REF), refObject).asObject();
+                        , executor
+                )
+                .thenApply(refObject ->
+                        JsonFactory.mergeJsonValues(objectWithTmRef.remove(TM_REF), refObject).asObject()
+                );
     }
 }
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingSkeletonGenerator.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingSkeletonGenerator.java
index 72d7f330e73..33b1db01561 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingSkeletonGenerator.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingSkeletonGenerator.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Executor;
 import java.util.function.Function;
 import java.util.stream.IntStream;
@@ -32,8 +33,8 @@
 import javax.annotation.concurrent.Immutable;
 
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
-import org.eclipse.ditto.internal.utils.akka.logging.DittoLogger;
 import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory;
+import org.eclipse.ditto.internal.utils.akka.logging.ThreadSafeDittoLogger;
 import org.eclipse.ditto.json.JsonArray;
 import org.eclipse.ditto.json.JsonArrayBuilder;
 import org.eclipse.ditto.json.JsonObject;
@@ -67,9 +68,11 @@
 import org.eclipse.ditto.wot.model.StringSchema;
 import org.eclipse.ditto.wot.model.ThingDefinitionInvalidException;
 import org.eclipse.ditto.wot.model.ThingModel;
+import org.eclipse.ditto.wot.model.TmOptional;
 import org.eclipse.ditto.wot.model.WotThingModelInvalidException;
 
 import akka.actor.ActorSystem;
+import akka.japi.Pair;
 
 /**
  * Default Ditto specific implementation of {@link WotThingSkeletonGenerator}.
@@ -77,15 +80,14 @@
 @Immutable
 final class DefaultWotThingSkeletonGenerator implements WotThingSkeletonGenerator {
 
-    private static final DittoLogger LOGGER = DittoLoggerFactory.getLogger(DefaultWotThingSkeletonGenerator.class);
+    private static final ThreadSafeDittoLogger LOGGER =
+            DittoLoggerFactory.getThreadSafeLogger(DefaultWotThingSkeletonGenerator.class);
 
     private static final String TM_EXTENDS = "tm:extends";
 
     private static final String TM_SUBMODEL = "tm:submodel";
     private static final String TM_SUBMODEL_INSTANCE_NAME = "instanceName";
 
-
-
     private final WotThingModelFetcher thingModelFetcher;
     private final Executor executor;
     private final WotThingModelExtensionResolver thingModelExtensionResolver;
@@ -97,76 +99,98 @@ final class DefaultWotThingSkeletonGenerator implements WotThingSkeletonGenerato
     }
 
     @Override
-    public Optional generateThingSkeleton(final ThingId thingId,
+    public CompletionStage> generateThingSkeleton(final ThingId thingId,
             final ThingModel thingModel,
             final URL thingModelUrl,
             final DittoHeaders dittoHeaders) {
 
-        final ThingModel thingModelWithExtensions = thingModelExtensionResolver
-                .resolveThingModelExtensions(thingModel, dittoHeaders);
-        final ThingModel thingModelWithExtensionsAndImports = thingModelExtensionResolver
-                .resolveThingModelRefs(thingModelWithExtensions, dittoHeaders);
-
-        final Optional dittoExtensionPrefix = thingModelWithExtensionsAndImports.getAtContext()
-                .determinePrefixFor(DittoWotExtension.DITTO_WOT_EXTENSION);
-
-        LOGGER.withCorrelationId(dittoHeaders)
-                .debug("ThingModel for generating Thing skeleton after resolving extensions + refs: <{}>",
-                        thingModelWithExtensionsAndImports);
-
-        final ThingBuilder.FromScratch builder = Thing.newBuilder();
-        thingModelWithExtensionsAndImports.getProperties()
-                .map(properties -> {
-                    final JsonObjectBuilder jsonObjectBuilder = JsonObject.newBuilder();
-                    final Map attributesCategories = new LinkedHashMap<>();
-
-                    fillPropertiesInOptionalCategories(properties, jsonObjectBuilder, attributesCategories,
-                            property -> dittoExtensionPrefix.flatMap(prefix ->
-                                    property.getValue(prefix + ":" + DittoWotExtension.DITTO_WOT_EXTENSION_CATEGORY)
+        return thingModelExtensionResolver
+                .resolveThingModelExtensions(thingModel, dittoHeaders)
+                .thenCompose(thingModelWithExtensions ->
+                        thingModelExtensionResolver.resolveThingModelRefs(thingModelWithExtensions, dittoHeaders)
+                )
+                .thenApply(thingModelWithExtensionsAndImports -> {
+                    final Optional dittoExtensionPrefix = thingModelWithExtensionsAndImports.getAtContext()
+                            .determinePrefixFor(DittoWotExtension.DITTO_WOT_EXTENSION);
+
+                    LOGGER.withCorrelationId(dittoHeaders)
+                            .debug("ThingModel for generating Thing skeleton after resolving extensions + refs: <{}>",
+                                    thingModelWithExtensionsAndImports);
+
+                    final ThingBuilder.FromScratch builder = Thing.newBuilder();
+                    thingModelWithExtensionsAndImports.getProperties()
+                            .map(properties -> {
+                                final JsonObjectBuilder jsonObjectBuilder = JsonObject.newBuilder();
+                                final Map attributesCategories = new LinkedHashMap<>();
+
+                                fillPropertiesInOptionalCategories(
+                                        properties,
+                                        thingModelWithExtensionsAndImports.getTmOptional().orElse(null),
+                                        jsonObjectBuilder,
+                                        attributesCategories,
+                                        property -> dittoExtensionPrefix.flatMap(prefix ->
+                                                        property.getValue(prefix + ":" +
+                                                                DittoWotExtension.DITTO_WOT_EXTENSION_CATEGORY
+                                                        )
+                                                )
+                                                .filter(JsonValue::isString)
+                                                .map(JsonValue::asString)
+                                );
+
+                                final AttributesBuilder attributesBuilder = Attributes.newBuilder();
+                                if (attributesCategories.size() > 0) {
+                                    attributesCategories.forEach((attributeCategory, categoryObjBuilder) ->
+                                            attributesBuilder.set(attributeCategory, categoryObjBuilder.build())
+                                    );
+                                }
+                                attributesBuilder.setAll(jsonObjectBuilder.build());
+                                return attributesBuilder.build();
+                            }).ifPresent(builder::setAttributes);
+
+                    return Pair.apply(thingModelWithExtensionsAndImports, builder);
+                })
+                .thenCompose(pair ->
+                    createFeaturesFromSubmodels(pair.first(), dittoHeaders)
+                            .thenApply(features ->
+                                    features.map(f -> pair.second().setFeatures(f)).orElse(pair.second())
                             )
-                            .filter(JsonValue::isString)
-                            .map(JsonValue::asString)
-                    );
-
-                    final AttributesBuilder attributesBuilder = Attributes.newBuilder();
-                    if (attributesCategories.size() > 0) {
-                        attributesCategories.forEach((attributeCategory, categoryObjBuilder) ->
-                                attributesBuilder.set(attributeCategory, categoryObjBuilder.build())
-                        );
-                    }
-                    attributesBuilder.setAll(jsonObjectBuilder.build());
-                    return attributesBuilder.build();
-                }).ifPresent(builder::setAttributes);
-
-        createFeaturesFromSubmodels(thingModelWithExtensionsAndImports, dittoHeaders)
-                .ifPresent(builder::setFeatures);
-
-        return Optional.of(builder.build());
+                )
+                .thenApply(builder -> Optional.of(builder.build()));
     }
 
     private static void fillPropertiesInOptionalCategories(final Properties properties,
+            @Nullable final TmOptional tmOptionalElements,
             final JsonObjectBuilder jsonObjectBuilder,
             final Map propertiesCategories,
             final Function> propertyCategoryExtractor) {
 
-        properties.values().forEach(property ->
-                determineInitialPropertyValue(property).ifPresent(val ->
-                                propertyCategoryExtractor.apply(property)
-                                .ifPresentOrElse(attributeCategory -> {
-                                        if (!propertiesCategories.containsKey(attributeCategory)) {
-                                            propertiesCategories.put(attributeCategory,
-                                                    JsonObject.newBuilder());
-                                        }
-                                        propertiesCategories.get(attributeCategory)
-                                                .set(property.getPropertyName(), val);
-                                }, () ->
-                                        jsonObjectBuilder.set(property.getPropertyName(), val)
+        properties.values().stream()
+                // filter out optional elements - don't create skeleton values for those:
+                .filter(property -> Optional.ofNullable(tmOptionalElements)
+                        .stream()
+                        .noneMatch(optionals -> optionals.stream()
+                                .anyMatch(optionalEl ->
+                                        optionalEl.toString().equals("/properties/" + property.getPropertyName())
                                 )
+                        )
                 )
-        );
+                .forEach(property -> determineInitialPropertyValue(property).ifPresent(val ->
+                                propertyCategoryExtractor.apply(property)
+                                        .ifPresentOrElse(attributeCategory -> {
+                                                    if (!propertiesCategories.containsKey(attributeCategory)) {
+                                                        propertiesCategories.put(attributeCategory,
+                                                                JsonObject.newBuilder());
+                                                    }
+                                                    propertiesCategories.get(attributeCategory)
+                                                            .set(property.getPropertyName(), val);
+                                                }, () ->
+                                                        jsonObjectBuilder.set(property.getPropertyName(), val)
+                                        )
+                        )
+                );
     }
 
-    private Optional createFeaturesFromSubmodels(final ThingModel thingModel,
+    private CompletionStage> createFeaturesFromSubmodels(final ThingModel thingModel,
             final DittoHeaders dittoHeaders) {
 
         final FeaturesBuilder featuresBuilder = Features.newBuilder();
@@ -192,32 +216,34 @@ private Optional createFeaturesFromSubmodels(final ThingModel thingMod
                 )
                 .orElseGet(Stream::empty)
                 .map(submodel -> thingModelFetcher.fetchThingModel(submodel.href, dittoHeaders)
-                        .thenApplyAsync(subThingModel ->
+                        .thenComposeAsync(subThingModel ->
                                 generateFeatureSkeleton(submodel.instanceName,
                                         subThingModel,
                                         submodel.href,
-                                        dittoHeaders), executor)
+                                        dittoHeaders
+                                ), executor)
                         .toCompletableFuture()
                 )
                 .toList();
 
-        final List features = CompletableFuture.allOf(futureList.toArray(CompletableFuture[]::new))
-                .thenApply(v -> futureList.stream()
-                        .map(CompletableFuture::join)
-                        .filter(Optional::isPresent)
-                        .map(Optional::get)
-                        .toList()
-                ).join();
-
-        if (features.isEmpty()) {
-            return Optional.empty();
-        } else {
-            featuresBuilder.setAll(features);
-            return Optional.of(featuresBuilder.build());
-        }
+        return CompletableFuture.allOf(futureList.toArray(CompletableFuture[]::new))
+                .thenApplyAsync(v -> {
+                            if (futureList.isEmpty()) {
+                                return Optional.empty();
+                            } else {
+                                featuresBuilder.setAll(futureList.stream()
+                                        .map(CompletableFuture::join)
+                                        .filter(Optional::isPresent)
+                                        .map(Optional::get)
+                                        .toList());
+                                return Optional.of(featuresBuilder.build());
+                            }
+                        },
+                        executor
+                );
     }
 
-    private Optional generateFeatureSkeleton(final String featureId,
+    private CompletionStage> generateFeatureSkeleton(final String featureId,
             final ThingModel thingModel,
             final IRI thingModelIri,
             final DittoHeaders dittoHeaders) {
@@ -231,50 +257,61 @@ private Optional generateFeatureSkeleton(final String featureId,
     }
 
     @Override
-    public Optional generateFeatureSkeleton(final String featureId,
+    public CompletionStage> generateFeatureSkeleton(final String featureId,
             final ThingModel thingModel,
             final URL thingModelUrl,
             final DittoHeaders dittoHeaders) {
 
-        final ThingModel thingModelWithExtensions = thingModelExtensionResolver
-                .resolveThingModelExtensions(thingModel, dittoHeaders);
-        final ThingModel thingModelWithExtensionsAndImports = thingModelExtensionResolver
-                .resolveThingModelRefs(thingModelWithExtensions, dittoHeaders);
-
-        final Optional dittoExtensionPrefix = thingModelWithExtensionsAndImports.getAtContext()
-                .determinePrefixFor(DittoWotExtension.DITTO_WOT_EXTENSION);
-
-        LOGGER.withCorrelationId(dittoHeaders)
-                .debug("ThingModel for generating Feature skeleton after resolving extensions + refs: <{}>",
-                        thingModelWithExtensionsAndImports);
-
-        final FeatureBuilder.FromScratchBuildable builder = Feature.newBuilder();
-        thingModelWithExtensionsAndImports.getProperties()
-                .map(properties -> {
-                    final JsonObjectBuilder jsonObjectBuilder = JsonObject.newBuilder();
-                    final Map propertiesCategories = new LinkedHashMap<>();
-
-                    fillPropertiesInOptionalCategories(properties, jsonObjectBuilder, propertiesCategories,
-                            property -> dittoExtensionPrefix.flatMap(prefix ->
-                                            property.getValue(prefix + ":" + DittoWotExtension.DITTO_WOT_EXTENSION_CATEGORY)
-                                    )
-                                    .filter(JsonValue::isString)
-                                    .map(JsonValue::asString)
-                    );
-
-                    final FeaturePropertiesBuilder propertiesBuilder = FeatureProperties.newBuilder();
-                    if (propertiesCategories.size() > 0) {
-                        propertiesCategories.forEach((propertyCategory, categoryObjBuilder) ->
-                                propertiesBuilder.set(propertyCategory, categoryObjBuilder.build())
-                        );
-                    }
-                    propertiesBuilder.setAll(jsonObjectBuilder.build());
-                    return propertiesBuilder.build();
-                }).ifPresent(builder::properties);
+        return thingModelExtensionResolver
+                .resolveThingModelExtensions(thingModel, dittoHeaders)
+                .thenCompose(thingModelWithExtensions -> thingModelExtensionResolver
+                        .resolveThingModelRefs(thingModelWithExtensions, dittoHeaders))
+                .thenCombine(resolveFeatureDefinition(thingModel, thingModelUrl, dittoHeaders),
+                        (thingModelWithExtensionsAndImports, featureDefinition) -> {
+                            final Optional dittoExtensionPrefix =
+                                    thingModelWithExtensionsAndImports.getAtContext()
+                                            .determinePrefixFor(DittoWotExtension.DITTO_WOT_EXTENSION);
+
+                            LOGGER.withCorrelationId(dittoHeaders)
+                                    .debug("ThingModel for generating Feature skeleton after resolving extensions + refs: <{}>",
+                                            thingModelWithExtensionsAndImports);
+
+                            final FeatureBuilder.FromScratchBuildable builder = Feature.newBuilder();
+                            thingModelWithExtensionsAndImports.getProperties()
+                                    .map(properties -> {
+                                        final JsonObjectBuilder jsonObjectBuilder = JsonObject.newBuilder();
+                                        final Map propertiesCategories =
+                                                new LinkedHashMap<>();
+
+                                        fillPropertiesInOptionalCategories(
+                                                properties,
+                                                thingModelWithExtensionsAndImports.getTmOptional().orElse(null),
+                                                jsonObjectBuilder,
+                                                propertiesCategories,
+                                                property -> dittoExtensionPrefix.flatMap(prefix ->
+                                                                property.getValue(
+                                                                        prefix + ":" +
+                                                                                DittoWotExtension.DITTO_WOT_EXTENSION_CATEGORY)
+                                                        )
+                                                        .filter(JsonValue::isString)
+                                                        .map(JsonValue::asString)
+                                        );
+
+                                        final FeaturePropertiesBuilder propertiesBuilder =
+                                                FeatureProperties.newBuilder();
+                                        if (propertiesCategories.size() > 0) {
+                                            propertiesCategories.forEach((propertyCategory, categoryObjBuilder) ->
+                                                    propertiesBuilder.set(propertyCategory, categoryObjBuilder.build())
+                                            );
+                                        }
+                                        propertiesBuilder.setAll(jsonObjectBuilder.build());
+                                        return propertiesBuilder.build();
+                                    }).ifPresent(builder::properties);
 
-        builder.definition(resolveFeatureDefinition(thingModel, thingModelUrl, dittoHeaders));
+                            builder.definition(featureDefinition);
 
-        return Optional.of(builder.withId(featureId).build());
+                            return Optional.of(builder.withId(featureId).build());
+                        });
     }
 
     private static Optional determineInitialPropertyValue(final SingleDataSchema dataSchema) {
@@ -429,14 +466,17 @@ private static String provideNeutralStringElement(@Nullable final Integer minLen
         return "";
     }
 
-    private FeatureDefinition resolveFeatureDefinition(final ThingModel thingModel, final URL thingModelUrl,
+    private CompletionStage resolveFeatureDefinition(final ThingModel thingModel, final URL thingModelUrl,
             final DittoHeaders dittoHeaders) {
-        return FeatureDefinition.fromIdentifier(thingModelUrl.toString(),
-                determineFurtherFeatureDefinitionIdentifiers(thingModel, dittoHeaders)
-                        .toArray(new DefinitionIdentifier[]{}));
+        return determineFurtherFeatureDefinitionIdentifiers(thingModel, dittoHeaders)
+                .thenApply(definitionIdentifiers -> FeatureDefinition.fromIdentifier(
+                        thingModelUrl.toString(),
+                        definitionIdentifiers.toArray(DefinitionIdentifier[]::new)
+                ));
     }
 
-    private List determineFurtherFeatureDefinitionIdentifiers(final ThingModel thingModel,
+    private CompletionStage> determineFurtherFeatureDefinitionIdentifiers(
+            final ThingModel thingModel,
             final DittoHeaders dittoHeaders) {
         return thingModel.getLinks().map(links -> {
             final Optional> extendsLink = links.stream()
@@ -445,23 +485,23 @@ private List determineFurtherFeatureDefinitionIdentifiers(
 
             if (extendsLink.isPresent()) {
                 final BaseLink link = extendsLink.get();
-                final List recursedSubmodels =
-                        thingModelFetcher.fetchThingModel(link.getHref(), dittoHeaders)
-                                .thenApplyAsync(subThingModel ->
-                                        determineFurtherFeatureDefinitionIdentifiers( // recurse!
-                                                subThingModel,
-                                                dittoHeaders
-                                        ), executor)
-                                .toCompletableFuture()
-                                .join();
-                final List combinedIdentifiers = new ArrayList<>();
-                combinedIdentifiers.add(ThingsModelFactory.newFeatureDefinitionIdentifier(link.getHref()));
-                combinedIdentifiers.addAll(recursedSubmodels);
-                return combinedIdentifiers;
+                return thingModelFetcher.fetchThingModel(link.getHref(), dittoHeaders)
+                        .thenComposeAsync(subThingModel ->
+                                determineFurtherFeatureDefinitionIdentifiers( // recurse!
+                                        subThingModel,
+                                        dittoHeaders
+                                ), executor
+                        )
+                        .thenApply(recursedSubmodels -> {
+                            final List combinedIdentifiers = new ArrayList<>();
+                            combinedIdentifiers.add(ThingsModelFactory.newFeatureDefinitionIdentifier(link.getHref()));
+                            combinedIdentifiers.addAll(recursedSubmodels);
+                            return combinedIdentifiers;
+                        });
             } else {
-                return Collections.emptyList();
+                return CompletableFuture.completedStage(Collections.emptyList());
             }
-        }).orElseGet(Collections::emptyList);
+        }).orElseGet(() -> CompletableFuture.completedStage(Collections.emptyList()));
     }
 
     private static class Submodel {
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingDescriptionGenerator.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingDescriptionGenerator.java
index 799d565e5a5..c40d8f1b571 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingDescriptionGenerator.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingDescriptionGenerator.java
@@ -13,6 +13,7 @@
 package org.eclipse.ditto.wot.integration.generator;
 
 import java.net.URL;
+import java.util.concurrent.CompletionStage;
 
 import javax.annotation.Nullable;
 
@@ -54,7 +55,7 @@ public interface WotThingDescriptionGenerator {
      * @throws org.eclipse.ditto.wot.model.WotThingModelInvalidException if the WoT ThingModel did not contain the
      * mandatory {@code "@type"} being {@code "tm:ThingModel"}
      */
-    ThingDescription generateThingDescription(ThingId thingId,
+    CompletionStage generateThingDescription(ThingId thingId,
             @Nullable Thing thing,
             @Nullable JsonObject placeholderLookupObject,
             @Nullable String featureId,
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingModelExtensionResolver.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingModelExtensionResolver.java
index 251602bc631..1cd749befe0 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingModelExtensionResolver.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingModelExtensionResolver.java
@@ -12,6 +12,8 @@
  */
 package org.eclipse.ditto.wot.integration.generator;
 
+import java.util.concurrent.CompletionStage;
+
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.wot.model.ThingModel;
 
@@ -37,7 +39,7 @@ public interface WotThingModelExtensionResolver {
      * @throws org.eclipse.ditto.wot.model.WotThingModelInvalidException if the fetched extended ThingModel could not be
      * parsed/interpreted as correct WoT ThingModel.
      */
-    ThingModel resolveThingModelExtensions(ThingModel thingModel, DittoHeaders dittoHeaders);
+    CompletionStage resolveThingModelExtensions(ThingModel thingModel, DittoHeaders dittoHeaders);
 
     /**
      * Resolves the "references" ({@code tm:ref}) contained in the passed {@code thingModel} and merges them into the
@@ -53,5 +55,5 @@ public interface WotThingModelExtensionResolver {
      * @throws org.eclipse.ditto.wot.model.WotThingModelInvalidException if the fetched referenced ThingModel could not
      * be parsed/interpreted as correct WoT ThingModel.
      */
-    ThingModel resolveThingModelRefs(ThingModel thingModel, DittoHeaders dittoHeaders);
+    CompletionStage resolveThingModelRefs(ThingModel thingModel, DittoHeaders dittoHeaders);
 }
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingSkeletonGenerator.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingSkeletonGenerator.java
index c61ca238459..b458ddb63bb 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingSkeletonGenerator.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/WotThingSkeletonGenerator.java
@@ -14,6 +14,7 @@
 
 import java.net.URL;
 import java.util.Optional;
+import java.util.concurrent.CompletionStage;
 
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
 import org.eclipse.ditto.things.model.Feature;
@@ -50,7 +51,7 @@ public interface WotThingSkeletonGenerator {
      * @throws org.eclipse.ditto.wot.model.WotThingModelInvalidException if the WoT ThingModel did not contain the
      * mandatory {@code "@type"} being {@code "tm:ThingModel"}
      */
-    Optional generateThingSkeleton(ThingId thingId,
+    CompletionStage> generateThingSkeleton(ThingId thingId,
             ThingModel thingModel,
             URL thingModelUrl,
             DittoHeaders dittoHeaders);
@@ -71,7 +72,7 @@ Optional generateThingSkeleton(ThingId thingId,
      * @throws org.eclipse.ditto.wot.model.WotThingModelInvalidException if the WoT ThingModel did not contain the
      * mandatory {@code "@type"} being {@code "tm:ThingModel"}
      */
-    Optional generateFeatureSkeleton(String featureId,
+    CompletionStage> generateFeatureSkeleton(String featureId,
             ThingModel thingModel,
             URL thingModelUrl,
             DittoHeaders dittoHeaders);
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/DefaultWotThingDescriptionProvider.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/DefaultWotThingDescriptionProvider.java
index e3bb23c02df..46e11581281 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/DefaultWotThingDescriptionProvider.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/DefaultWotThingDescriptionProvider.java
@@ -16,7 +16,9 @@
 
 import java.net.URL;
 import java.util.Optional;
-import java.util.concurrent.CompletionException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.Executor;
 
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
@@ -61,12 +63,14 @@ final class DefaultWotThingDescriptionProvider implements WotThingDescriptionPro
     private final WotThingModelFetcher thingModelFetcher;
     private final WotThingDescriptionGenerator thingDescriptionGenerator;
     private final WotThingSkeletonGenerator thingSkeletonGenerator;
+    private final Executor executor;
 
     private DefaultWotThingDescriptionProvider(final ActorSystem actorSystem, final WotConfig wotConfig) {
         this.wotConfig = checkNotNull(wotConfig, "wotConfig");
         thingModelFetcher = new DefaultWotThingModelFetcher(actorSystem, wotConfig);
         thingDescriptionGenerator = WotThingDescriptionGenerator.of(actorSystem, wotConfig, thingModelFetcher);
         thingSkeletonGenerator = WotThingSkeletonGenerator.of(actorSystem, thingModelFetcher);
+        executor = actorSystem.dispatchers().lookup("wot-dispatcher");
     }
 
     /**
@@ -82,7 +86,7 @@ public static DefaultWotThingDescriptionProvider of(final ActorSystem actorSyste
     }
 
     @Override
-    public ThingDescription provideThingTD(@Nullable final ThingDefinition thingDefinition,
+    public CompletionStage provideThingTD(@Nullable final ThingDefinition thingDefinition,
             final ThingId thingId,
             @Nullable final Thing thing,
             final DittoHeaders dittoHeaders) {
@@ -96,7 +100,7 @@ public ThingDescription provideThingTD(@Nullable final ThingDefinition thingDefi
     }
 
     @Override
-    public ThingDescription provideFeatureTD(final ThingId thingId,
+    public CompletionStage provideFeatureTD(final ThingId thingId,
             @Nullable final Thing thing,
             final Feature feature,
             final DittoHeaders dittoHeaders) {
@@ -112,7 +116,7 @@ public ThingDescription provideFeatureTD(final ThingId thingId,
     }
 
     @Override
-    public Optional provideThingSkeletonForCreation(final ThingId thingId,
+    public CompletionStage> provideThingSkeletonForCreation(final ThingId thingId,
             @Nullable final ThingDefinition thingDefinition,
             final DittoHeaders dittoHeaders) {
 
@@ -123,31 +127,35 @@ public Optional provideThingSkeletonForCreation(final ThingId thingId,
             final Optional urlOpt = thingDefinition.getUrl();
             if (urlOpt.isPresent()) {
                 final URL url = urlOpt.get();
-                try {
-                    logger.debug("Fetching ThingModel from <{}> in order to create Thing skeleton for new Thing " +
-                            "with id <{}>", url, thingId);
-                    final Optional thingSkeleton = thingModelFetcher.fetchThingModel(url, dittoHeaders)
-                            .thenApply(thingModel -> thingSkeletonGenerator
-                                    .generateThingSkeleton(thingId, thingModel, url, dittoHeaders))
-                            .toCompletableFuture()
-                            .join();
-                    logger.debug("Created Thing skeleton for new Thing with id <{}>: <{}>", thingId, thingSkeleton);
-                    return thingSkeleton;
-                } catch (final DittoRuntimeException | CompletionException e) {
-                    logger.info("Could not fetch ThingModel or generate Feature skeleton based on it due to: <{}: {}>",
-                            e.getClass().getSimpleName(), e.getMessage(), e);
-                    return Optional.empty();
-                }
+                logger.debug("Fetching ThingModel from <{}> in order to create Thing skeleton for new Thing " +
+                        "with id <{}>", url, thingId);
+                return thingModelFetcher.fetchThingModel(url, dittoHeaders)
+                        .thenComposeAsync(thingModel -> thingSkeletonGenerator
+                                        .generateThingSkeleton(thingId, thingModel, url, dittoHeaders),
+                                executor
+                        )
+                        .thenApply(thingSkeleton -> {
+                            logger.debug("Created Thing skeleton for new Thing with id <{}>: <{}>", thingId,
+                                    thingSkeleton);
+                            return thingSkeleton;
+                        })
+                        .exceptionally(throwable -> {
+                            logger.info("Could not fetch ThingModel or generate Thing skeleton based on it due " +
+                                            "to: <{}: {}>",
+                                    throwable.getClass().getSimpleName(),throwable.getMessage(), throwable);
+                            return Optional.empty();
+                        });
+
             } else {
-                return Optional.empty();
+                return CompletableFuture.completedStage(Optional.empty());
             }
         } else {
-            return Optional.empty();
+            return CompletableFuture.completedStage(Optional.empty());
         }
     }
 
     @Override
-    public Optional provideFeatureSkeletonForCreation(final String featureId,
+    public CompletionStage> provideFeatureSkeletonForCreation(final String featureId,
             @Nullable final FeatureDefinition featureDefinition, final DittoHeaders dittoHeaders) {
 
         final ThreadSafeDittoLogger logger = LOGGER.withCorrelationId(dittoHeaders);
@@ -157,34 +165,36 @@ public Optional provideFeatureSkeletonForCreation(final String featureI
             final Optional urlOpt = featureDefinition.getFirstIdentifier().getUrl();
             if (urlOpt.isPresent()) {
                 final URL url = urlOpt.get();
-                try {
-                    logger.debug("Fetching ThingModel from <{}> in order to create Feature skeleton for new Feature " +
-                            "with id <{}>", url, featureId);
-                    final Optional featureSkeleton = thingModelFetcher.fetchThingModel(url, dittoHeaders)
-                            .thenApply(thingModel -> thingSkeletonGenerator
-                                    .generateFeatureSkeleton(featureId, thingModel, url, dittoHeaders))
-                            .toCompletableFuture()
-                            .join();
-                    logger.debug("Created Feature skeleton for new Feature with id <{}>: <{}>", featureId,
-                            featureSkeleton);
-                    return featureSkeleton;
-                } catch (final DittoRuntimeException | CompletionException e) {
-                    logger.info("Could not fetch ThingModel or generate Feature skeleton based on it due to: <{}: {}>",
-                            e.getClass().getSimpleName(), e.getMessage(), e);
-                    return Optional.empty();
-                }
+                logger.debug("Fetching ThingModel from <{}> in order to create Feature skeleton for new Feature " +
+                        "with id <{}>", url, featureId);
+                return thingModelFetcher.fetchThingModel(url, dittoHeaders)
+                        .thenComposeAsync(thingModel -> thingSkeletonGenerator
+                                        .generateFeatureSkeleton(featureId, thingModel, url, dittoHeaders),
+                                executor
+                        )
+                        .thenApply(featureSkeleton -> {
+                            logger.debug("Created Feature skeleton for new Feature with id <{}>: <{}>", featureId,
+                                    featureSkeleton);
+                            return featureSkeleton;
+                        })
+                        .exceptionally(throwable -> {
+                            logger.info("Could not fetch ThingModel or generate Feature skeleton based on it " +
+                                            "due to: <{}: {}>",
+                                    throwable.getClass().getSimpleName(), throwable.getMessage(), throwable);
+                            return Optional.empty();
+                        });
             } else {
-                return Optional.empty();
+                return CompletableFuture.completedStage(Optional.empty());
             }
         } else {
-            return Optional.empty();
+            return CompletableFuture.completedStage(Optional.empty());
         }
     }
 
     /**
      * Download TM, add it to local cache + build TD + return it
      */
-    private ThingDescription getWotThingDescriptionForThing(final ThingDefinition definitionIdentifier,
+    private CompletionStage getWotThingDescriptionForThing(final ThingDefinition definitionIdentifier,
             final ThingId thingId,
             @Nullable final Thing thing,
             final DittoHeaders dittoHeaders) {
@@ -192,32 +202,30 @@ private ThingDescription getWotThingDescriptionForThing(final ThingDefinition de
         final Optional urlOpt = definitionIdentifier.getUrl();
         if (urlOpt.isPresent()) {
             final URL url = urlOpt.get();
-            try {
-                return thingModelFetcher.fetchThingModel(url, dittoHeaders)
-                        .thenApply(thingModel -> thingDescriptionGenerator
-                                .generateThingDescription(thingId,
-                                        thing,
-                                        Optional.ofNullable(thing)
-                                                .flatMap(Thing::getAttributes)
-                                                .flatMap(a -> a.getValue(MODEL_PLACEHOLDERS_KEY))
-                                                .filter(JsonValue::isObject)
-                                                .map(JsonValue::asObject)
-                                                .orElse(null),
-                                        null,
-                                        thingModel,
-                                        url,
-                                        dittoHeaders)
-                        )
-                        .toCompletableFuture()
-                        .join();
-            } catch (final Exception e) {
-                throw DittoRuntimeException.asDittoRuntimeException(e, throwable ->
-                        WotInternalErrorException.newBuilder()
-                                .dittoHeaders(dittoHeaders)
-                                .cause(e)
-                                .build()
-                );
-            }
+            return thingModelFetcher.fetchThingModel(url, dittoHeaders)
+                    .thenComposeAsync(thingModel -> thingDescriptionGenerator
+                            .generateThingDescription(thingId,
+                                    thing,
+                                    Optional.ofNullable(thing)
+                                            .flatMap(Thing::getAttributes)
+                                            .flatMap(a -> a.getValue(MODEL_PLACEHOLDERS_KEY))
+                                            .filter(JsonValue::isObject)
+                                            .map(JsonValue::asObject)
+                                            .orElse(null),
+                                    null,
+                                    thingModel,
+                                    url,
+                                    dittoHeaders
+                            ),
+                            executor
+                    )
+                    .exceptionally(throwable -> {
+                        throw DittoRuntimeException.asDittoRuntimeException(throwable, t ->
+                                WotInternalErrorException.newBuilder()
+                                        .dittoHeaders(dittoHeaders)
+                                        .cause(t)
+                                        .build());
+                    });
         } else {
             throw ThingDefinitionInvalidException.newBuilder(definitionIdentifier)
                     .dittoHeaders(dittoHeaders)
@@ -228,7 +236,7 @@ private ThingDescription getWotThingDescriptionForThing(final ThingDefinition de
     /**
      * Download TM, add it to local cache + build TD + return it
      */
-    private ThingDescription getWotThingDescriptionForFeature(final ThingId thingId,
+    private CompletionStage getWotThingDescriptionForFeature(final ThingId thingId,
             @Nullable final Thing thing,
             final Feature feature,
             final DittoHeaders dittoHeaders) {
@@ -238,31 +246,29 @@ private ThingDescription getWotThingDescriptionForFeature(final ThingId thingId,
         final Optional urlOpt = definitionIdentifier.flatMap(DefinitionIdentifier::getUrl);
         if (urlOpt.isPresent()) {
             final URL url = urlOpt.get();
-            try {
-                return thingModelFetcher.fetchThingModel(url, dittoHeaders)
-                        .thenApply(thingModel -> thingDescriptionGenerator
-                                .generateThingDescription(thingId,
-                                        thing,
-                                        feature.getProperties()
-                                                .flatMap(p -> p.getValue(MODEL_PLACEHOLDERS_KEY))
-                                                .filter(JsonValue::isObject)
-                                                .map(JsonValue::asObject)
-                                                .orElse(null),
-                                        feature.getId(),
-                                        thingModel,
-                                        url,
-                                        dittoHeaders)
-                        )
-                        .toCompletableFuture()
-                        .join();
-            } catch (final Exception e) {
-                throw DittoRuntimeException.asDittoRuntimeException(e, throwable ->
-                        WotInternalErrorException.newBuilder()
-                                .dittoHeaders(dittoHeaders)
-                                .cause(e)
-                                .build()
-                );
-            }
+            return thingModelFetcher.fetchThingModel(url, dittoHeaders)
+                    .thenComposeAsync(thingModel -> thingDescriptionGenerator
+                            .generateThingDescription(thingId,
+                                    thing,
+                                    feature.getProperties()
+                                            .flatMap(p -> p.getValue(MODEL_PLACEHOLDERS_KEY))
+                                            .filter(JsonValue::isObject)
+                                            .map(JsonValue::asObject)
+                                            .orElse(null),
+                                    feature.getId(),
+                                    thingModel,
+                                    url,
+                                    dittoHeaders
+                            ),
+                            executor
+                    )
+                    .exceptionally(throwable -> {
+                        throw DittoRuntimeException.asDittoRuntimeException(throwable, t ->
+                                WotInternalErrorException.newBuilder()
+                                        .dittoHeaders(dittoHeaders)
+                                        .cause(t)
+                                        .build());
+                    });
         } else {
             throw ThingDefinitionInvalidException.newBuilder(definitionIdentifier.orElse(null))
                     .dittoHeaders(dittoHeaders)
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/DefaultWotThingModelFetcher.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/DefaultWotThingModelFetcher.java
index 365e750ee33..f9c8406b389 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/DefaultWotThingModelFetcher.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/DefaultWotThingModelFetcher.java
@@ -15,10 +15,12 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.text.MessageFormat;
+import java.time.Duration;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
 
 import javax.annotation.Nullable;
 
@@ -63,6 +65,8 @@ final class DefaultWotThingModelFetcher implements WotThingModelFetcher {
     private static final ThreadSafeDittoLogger LOGGER =
             DittoLoggerFactory.getThreadSafeLogger(DefaultWotThingModelFetcher.class);
 
+    private static final Duration MAX_FETCH_MODEL_DURATION = Duration.ofSeconds(10);
+
     private static final HttpHeader ACCEPT_HEADER = Accept.create(
             MediaRanges.create(MediaTypes.applicationWithOpenCharset("tm+json")),
             MediaRanges.create(MediaTypes.APPLICATION_JSON)
@@ -79,7 +83,7 @@ final class DefaultWotThingModelFetcher implements WotThingModelFetcher {
         thingModelCache = CacheFactory.createCache(loader,
                 wotConfig.getCacheConfig(),
                 "ditto_wot_thing_model_cache",
-                actorSystem.dispatchers().lookup("wot-dispatcher"));
+                actorSystem.dispatchers().lookup("wot-dispatcher-cache-loader"));
     }
 
     @Override
@@ -98,7 +102,8 @@ public CompletableFuture fetchThingModel(final URL url, final DittoH
         LOGGER.withCorrelationId(dittoHeaders)
                 .debug("Fetching ThingModel (from cache or downloading as fallback) from URL: <{}>", url);
         return thingModelCache.get(url)
-                .thenApply(optTm -> resolveThingModel(optTm.orElse(null), url, dittoHeaders));
+                .thenApply(optTm -> resolveThingModel(optTm.orElse(null), url, dittoHeaders))
+                .orTimeout(MAX_FETCH_MODEL_DURATION.toSeconds(), TimeUnit.SECONDS);
     }
 
     private ThingModel resolveThingModel(@Nullable final ThingModel thingModel,
diff --git a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/WotThingDescriptionProvider.java b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/WotThingDescriptionProvider.java
index da3da7f2c7d..72ca8c754b2 100644
--- a/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/WotThingDescriptionProvider.java
+++ b/wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/WotThingDescriptionProvider.java
@@ -13,6 +13,7 @@
 package org.eclipse.ditto.wot.integration.provider;
 
 import java.util.Optional;
+import java.util.concurrent.CompletionStage;
 
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
@@ -52,7 +53,7 @@ public interface WotThingDescriptionProvider extends Extension {
      * @throws org.eclipse.ditto.wot.model.WotThingModelNotAccessibleException if the ThingModel could not be accessed/
      * downloaded.
      */
-    ThingDescription provideThingTD(@Nullable ThingDefinition thingDefinition,
+    CompletionStage provideThingTD(@Nullable ThingDefinition thingDefinition,
             ThingId thingId,
             @Nullable Thing thing,
             DittoHeaders dittoHeaders);
@@ -71,7 +72,7 @@ ThingDescription provideThingTD(@Nullable ThingDefinition thingDefinition,
      * @throws org.eclipse.ditto.wot.model.WotThingModelNotAccessibleException if the ThingModel could not be accessed/
      * downloaded.
      */
-    ThingDescription provideFeatureTD(ThingId thingId,
+    CompletionStage provideFeatureTD(ThingId thingId,
             @Nullable Thing thing,
             Feature feature,
             DittoHeaders dittoHeaders);
@@ -88,7 +89,7 @@ ThingDescription provideFeatureTD(ThingId thingId,
      * @param dittoHeaders the DittoHeaders for possibly thrown DittoRuntimeExceptions.
      * @return an optional Thing skeleton or empty optional if something went wrong during the skeleton creation.
      */
-    Optional provideThingSkeletonForCreation(ThingId thingId,
+    CompletionStage> provideThingSkeletonForCreation(ThingId thingId,
             @Nullable ThingDefinition thingDefinition,
             DittoHeaders dittoHeaders);
 
@@ -104,7 +105,7 @@ Optional provideThingSkeletonForCreation(ThingId thingId,
      * @param dittoHeaders the DittoHeaders for possibly thrown DittoRuntimeExceptions.
      * @return an optional Feature skeleton or empty optional if something went wrong during the skeleton creation.
      */
-    Optional provideFeatureSkeletonForCreation(String featureId,
+    CompletionStage> provideFeatureSkeletonForCreation(String featureId,
             @Nullable FeatureDefinition featureDefinition,
             DittoHeaders dittoHeaders);
 
diff --git a/wot/model/README.md b/wot/model/README.md
index 0677705a4f1..9384c1e7608 100755
--- a/wot/model/README.md
+++ b/wot/model/README.md
@@ -106,7 +106,7 @@ public final class TesterBuilder {
                         Arrays.asList(
                                 SingleUriAtContext.W3ORG_2022_WOT_TD_V11,
                                 SinglePrefixedAtContext.of("ditto",
-                                        SingleUriAtContext.of("https://www.eclipse.org/ditto/ctx"))
+                                        SingleUriAtContext.of("https://www.eclipse.dev/ditto/ctx"))
                         )
                 ))
                 .setId(IRI.of("urn:org.eclipse.ditto:333-WoTLamp-1234"))
diff --git a/wot/model/src/test/java/org/eclipse/ditto/wot/model/ThingDescriptionTest.java b/wot/model/src/test/java/org/eclipse/ditto/wot/model/ThingDescriptionTest.java
index 27611f533b4..793f29364a8 100644
--- a/wot/model/src/test/java/org/eclipse/ditto/wot/model/ThingDescriptionTest.java
+++ b/wot/model/src/test/java/org/eclipse/ditto/wot/model/ThingDescriptionTest.java
@@ -69,7 +69,7 @@ public void testOverallParsingOfThingDescriptionFromJson() {
         assertThat(thingDescription.getAtContext()).isEqualTo(MultipleAtContext.of(
                 Arrays.asList(
                         SingleUriAtContext.W3ORG_2022_WOT_TD_V11,
-                        SinglePrefixedAtContext.of("ditto", SingleUriAtContext.of("https://www.eclipse.org/ditto/ctx")),
+                        SinglePrefixedAtContext.of("ditto", SingleUriAtContext.of("https://www.eclipse.dev/ditto/ctx")),
                         SinglePrefixedAtContext.of("ace", SingleUriAtContext.of("http://www.example.org/ace-security#"))
                 )
         ));
@@ -137,7 +137,7 @@ public void testOverallParsingOfThingDescriptionFromJson() {
         final List> expectedLinks = new ArrayList<>();
         expectedLinks.add(Link.newBuilder()
                 .setRel("service-doc")
-                .setHref(IRI.of("https://eclipse.org/ditto/some-pdf.pdf"))
+                .setHref(IRI.of("https://eclipse.dev/ditto/some-pdf.pdf"))
                 .setType("application/pdf")
                 .setHreflang(Hreflang.newSingleHreflang("de-CH-1996"))
                 .build()
@@ -161,7 +161,7 @@ public void testBuildingThingDescriptionWithBuilder() {
                         Arrays.asList(
                                 SingleUriAtContext.W3ORG_2022_WOT_TD_V11,
                                 SinglePrefixedAtContext.of("ditto",
-                                        SingleUriAtContext.of("https://www.eclipse.org/ditto/ctx")),
+                                        SingleUriAtContext.of("https://www.eclipse.dev/ditto/ctx")),
                                 SinglePrefixedAtContext.of("ace",
                                         SingleUriAtContext.of("http://www.example.org/ace-security#"))
                         )
@@ -203,7 +203,7 @@ public void testBuildingThingDescriptionWithBuilder() {
                 )))
                 .setLinks(Links.of(Collections.singletonList(Link.newBuilder()
                         .setRel("service-doc")
-                        .setHref(IRI.of("https://eclipse.org/ditto/some-pdf.pdf"))
+                        .setHref(IRI.of("https://eclipse.dev/ditto/some-pdf.pdf"))
                         .setType("application/pdf")
                         .setHreflang(Hreflang.newSingleHreflang("de-CH-1996"))
                         .build()
diff --git a/wot/model/src/test/resources/tds/some-example.td.json b/wot/model/src/test/resources/tds/some-example.td.json
index 249530b8129..717b836a712 100644
--- a/wot/model/src/test/resources/tds/some-example.td.json
+++ b/wot/model/src/test/resources/tds/some-example.td.json
@@ -2,7 +2,7 @@
   "@context": [
     "https://www.w3.org/2022/wot/td/v1.1",
     {
-      "ditto": "https://www.eclipse.org/ditto/ctx",
+      "ditto": "https://www.eclipse.dev/ditto/ctx",
       "ace": "http://www.example.org/ace-security#"
     }
   ],
@@ -46,7 +46,7 @@
   },
   "links": [{
     "rel": "service-doc",
-    "href": "https://eclipse.org/ditto/some-pdf.pdf",
+    "href": "https://eclipse.dev/ditto/some-pdf.pdf",
     "type": "application/pdf",
     "hreflang" : "de-CH-1996"
   }]