From b569fc7f40727163b56a81dada95b07b67f3d524 Mon Sep 17 00:00:00 2001 From: thfries Date: Sun, 30 Oct 2022 20:02:44 +0100 Subject: [PATCH 001/173] Refactored things.js and split into smaller files - Enabled visual code type checking Signed-off-by: thfries --- ui/jsconfig.json | 10 + ui/main.js | 4 + ui/modules/things/searchFilter.js | 53 +---- ui/modules/things/things.js | 344 ++---------------------------- ui/modules/things/thingsCRUD.js | 168 +++++++++++++++ ui/modules/things/thingsSearch.js | 260 ++++++++++++++++++++++ 6 files changed, 469 insertions(+), 370 deletions(-) create mode 100644 ui/jsconfig.json create mode 100644 ui/modules/things/thingsCRUD.js create mode 100644 ui/modules/things/thingsSearch.js diff --git a/ui/jsconfig.json b/ui/jsconfig.json new file mode 100644 index 00000000000..0c1857a8d62 --- /dev/null +++ b/ui/jsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "module": "es6", + "target": "es6", + "lib": [ + "es2016", "DOM" + ] + }, + "exclude": ["node_modules", "**/node_modules/*"] +} \ No newline at end of file diff --git a/ui/main.js b/ui/main.js index 0b4540c0406..25855cec186 100644 --- a/ui/main.js +++ b/ui/main.js @@ -20,6 +20,8 @@ import * as FeatureMessages from './modules/things/featureMessages.js'; import * as Fields from './modules/things/fields.js'; import * as SearchFilter from './modules/things/searchFilter.js'; import * as Things from './modules/things/things.js'; +import * as ThingsSearch from './modules/things/thingsSearch.js'; +import * as ThingsCRUD from './modules/things/thingsCRUD.js'; import * as Connections from './modules/connections/connections.js'; import * as Policies from './modules/policies/policies.js'; import * as API from './modules/api.js'; @@ -44,6 +46,8 @@ document.addEventListener('DOMContentLoaded', async function() { Utils.ready(); await Things.ready(); + ThingsSearch.ready(); + ThingsCRUD.ready(); Attributes.ready(); await Fields.ready(); await SearchFilter.ready(); diff --git a/ui/modules/things/searchFilter.js b/ui/modules/things/searchFilter.js index be48d2aafe2..6eabd8b6086 100644 --- a/ui/modules/things/searchFilter.js +++ b/ui/modules/things/searchFilter.js @@ -10,10 +10,11 @@ * * SPDX-License-Identifier: EPL-2.0 */ +// @ts-check import * as Environments from '../environments/environments.js'; import * as Utils from '../utils.js'; -import * as Things from './things.js'; +import * as ThingsSearch from './thingsSearch.js'; const filterExamples = [ 'eq(attributes/location,"kitchen")', @@ -27,8 +28,6 @@ const filterExamples = [ let keyStrokeTimeout; -let lastSearch = ''; - const dom = { filterList: null, favIcon: null, @@ -47,16 +46,19 @@ export async function ready() { Utils.getAllElementsById(dom); + dom.pinnedThings.onclick = ThingsSearch.pinnedTriggered; + + dom.filterList.addEventListener('click', (event) => { if (event.target && event.target.classList.contains('dropdown-item')) { dom.searchFilterEdit.value = event.target.textContent; checkIfFavourite(); - Things.searchThings(event.target.textContent); + ThingsSearch.searchThings(event.target.textContent); } }); dom.searchThings.onclick = () => { - searchTriggered(dom.searchFilterEdit.value); + ThingsSearch.searchTriggered(dom.searchFilterEdit.value); }; dom.searchFavourite.onclick = () => { @@ -68,7 +70,7 @@ export async function ready() { dom.searchFilterEdit.onkeyup = (event) => { if (event.key === 'Enter' || event.code === 13) { - searchTriggered(dom.searchFilterEdit.value); + ThingsSearch.searchTriggered(dom.searchFilterEdit.value); } else { clearTimeout(keyStrokeTimeout); keyStrokeTimeout = setTimeout(checkIfFavourite, 1000); @@ -81,7 +83,9 @@ export async function ready() { } }; - dom.pinnedThings.onclick = pinnedTriggered; + dom.searchFilterEdit.onchange = ThingsSearch.removeMoreFromThingList; + + dom.searchFilterEdit.focus(); } /** @@ -97,41 +101,6 @@ function onEnvironmentChanged() { updateFilterList(); } -/** - * Tests if the search filter is an RQL. If yes, things search is called otherwise just things get - * @param {String} filter search filter string containing an RQL or a thingId - */ -function searchTriggered(filter) { - lastSearch = filter; - const regex = /^(eq\(|ne\(|gt\(|ge\(|lt\(|le\(|in\(|like\(|exists\(|and\(|or\(|not\().*/; - if (filter === '' || regex.test(filter)) { - Things.searchThings(filter); - } else { - Things.getThings([filter]); - } -} - -/** - * Gets the list of pinned things - */ -function pinnedTriggered() { - lastSearch = 'pinned'; - dom.searchFilterEdit.value = null; - dom.favIcon.classList.replace('bi-star-fill', 'bi-star'); - Things.getThings(Environments.current()['pinnedThings']); -} - -/** - * Performs the last search by the user using the last used filter. - * If the user used pinned things last time, the pinned things are reloaded - */ -export function performLastSearch() { - if (lastSearch === 'pinned') { - pinnedTriggered(); - } else { - searchTriggered(lastSearch); - } -} /** * Updates the UI filterList diff --git a/ui/modules/things/things.js b/ui/modules/things/things.js index da397b1a179..e8fff1e728c 100644 --- a/ui/modules/things/things.js +++ b/ui/modules/things/things.js @@ -1,54 +1,32 @@ -/* eslint-disable require-jsdoc */ /* - * Copyright (c) 2022 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ - -/* eslint-disable new-cap */ -/* eslint-disable no-invalid-this */ -import {JSONPath} from 'https://cdn.jsdelivr.net/npm/jsonpath-plus@5.0.3/dist/index-browser-esm.min.js'; +* Copyright (c) 2022 Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Eclipse Public License 2.0 which is available at +* http://www.eclipse.org/legal/epl-2.0 +* +* SPDX-License-Identifier: EPL-2.0 +*/ +/* eslint-disable require-jsdoc */ +// @ts-check import * as API from '../api.js'; import * as Environments from '../environments/environments.js'; -import * as SearchFilter from './searchFilter.js'; import * as Utils from '../utils.js'; -import * as Fields from './fields.js'; +import * as ThingsSearch from './thingsSearch.js'; export let theThing; -let theSearchCursor; - -let thingJsonEditor; - -let thingTemplates; const observers = []; const dom = { - thingsTableHead: null, - thingsTableBody: null, - thingDetails: null, - thingId: null, - buttonCreateThing: null, - buttonSaveThing: null, - buttonDeleteThing: null, - inputThingDefinition: null, - ulThingDefinitions: null, - tabModifyThing: null, - searchFilterEdit: null, collapseThings: null, tabThings: null, }; -const uuidRegex = /([0-9a-f]{7})[0-9a-f]-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g; - /** * Adds a listener function for the currently selected thing * @param {function} observer function that will be called if the current thing was changed @@ -65,216 +43,7 @@ export async function ready() { Utils.getAllElementsById(dom); - thingJsonEditor = Utils.createAceEditor('thingJsonEditor', 'ace/mode/json'); - - loadThingTemplates(); - - dom.ulThingDefinitions.addEventListener('click', (event) => { - setTheThing(null); - Utils.tableAdjustSelection(dom.thingsTableBody, () => false); - dom.inputThingDefinition.value = event.target.textContent; - thingJsonEditor.setValue(JSON.stringify(thingTemplates[event.target.textContent], null, 2), -1); - }); - - dom.searchFilterEdit.onchange = removeMoreFromThingList; - - dom.buttonCreateThing.onclick = async () => { - const editorValue = thingJsonEditor.getValue(); - if (dom.thingId.value !== undefined && dom.thingId.value !== '') { - API.callDittoREST('PUT', - '/things/' + dom.thingId.value, - editorValue === '' ? {} : JSON.parse(editorValue), - { - 'if-none-match': '*', - }, - ).then((data) => { - refreshThing(data.thingId, () => { - getThings([data.thingId]); - }); - }); - } else { - API.callDittoREST('POST', '/things', editorValue === '' ? {} : JSON.parse(editorValue)) - .then((data) => { - refreshThing(data.thingId, () => { - getThings([data.thingId]); - }); - }); - } - }; - - dom.buttonSaveThing.onclick = () => { - Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId); - modifyThing('PUT'); - }; - - dom.buttonDeleteThing.onclick = () => { - Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId); - Utils.confirm(`Are you sure you want to delete thing
'${theThing.thingId}'?`, 'Delete', () => { - modifyThing('DELETE'); - }); - }; - - dom.thingsTableBody.addEventListener('click', (event) => { - if (event.target && event.target.nodeName === 'TD') { - const row = event.target.parentNode; - if (row.id === 'searchThingsMore') { - row.style.pointerEvents = 'none'; - event.stopImmediatePropagation(); - searchThings(dom.searchFilterEdit.value, theSearchCursor); - } else { - if (theThing && theThing.thingId === row.id) { - setTheThing(null); - } else { - refreshThing(row.id); - } - } - } - }); - - document.querySelector('a[data-bs-target="#tabModifyThing"]').addEventListener('shown.bs.tab', (event) => { - thingJsonEditor.renderer.updateFull(); - }); - dom.tabThings.onclick = onTabActivated; - - dom.searchFilterEdit.focus(); -} - -function loadThingTemplates() { - fetch('templates/thingTemplates.json') - .then((response) => { - response.json().then((loadedTemplates) => { - thingTemplates = loadedTemplates; - Utils.addDropDownEntries(dom.ulThingDefinitions, Object.keys(thingTemplates)); - }); - }); -} - -/** - * Fills the things table UI with the given things - * @param {Array} thingsList Array of thing json objects - */ -function fillThingsTable(thingsList) { - const activeFields = Environments.current().fieldList.filter((f) => f.active); - fillHeaderRow(); - let thingSelected = false; - thingsList.forEach((item, t) => { - const row = dom.thingsTableBody.insertRow(); - fillBodyRow(row, item); - }); - if (!thingSelected) { - setTheThing(null); - } - - function fillHeaderRow() { - dom.thingsTableHead.innerHTML = ''; - // Utils.addCheckboxToRow(dom.thingsTableHead, 'checkboxHead', false, null); - Utils.insertHeaderCell(dom.thingsTableHead, ''); - Utils.insertHeaderCell(dom.thingsTableHead, 'Thing ID'); - activeFields.forEach((field) => { - Utils.insertHeaderCell(dom.thingsTableHead, field['label'] ? field.label : field.path); - }); - } - - function fillBodyRow(row, item) { - row.id = item.thingId; - if (theThing && (item.thingId === theThing.thingId)) { - thingSelected = true; - row.classList.add('table-active'); - } - Utils.addCheckboxToRow( - row, - item.thingId, - Environments.current().pinnedThings.includes(item.thingId), - togglePinnedThing, - ); - Utils.addCellToRow(row, beautifyId(item.thingId), item.thingId); - activeFields.forEach((field) => { - let path = field.path.replace(/\//g, '.'); - if (path.charAt(0) !== '.') { - path = '$.' + path; - } - const elem = JSONPath({ - json: item, - path: path, - }); - Utils.addCellToRow(row, elem.length !== 0 ? elem[0] : ''); - }); - } - - function beautifyId(longId) { - let result = longId; - if (Environments.current()['shortenUUID']) { - result = result.replace(uuidRegex, '$1'); - } - if (Environments.current()['defaultNamespace']) { - result = result.replace(Environments.current()['defaultNamespace'], 'dn'); - } - return result; - } -} - -/** - * Calls Ditto search api and fills UI with the result - * @param {String} filter Ditto search filter (rql) - * @param {String} cursor (optional) cursor returned from things search for additional pages - */ -export function searchThings(filter, cursor) { - document.body.style.cursor = 'progress'; - - const namespaces = Environments.current().searchNamespaces; - - API.callDittoREST('GET', - '/search/things?' + Fields.getQueryParameter() + - ((filter && filter !== '') ? '&filter=' + encodeURIComponent(filter) : '') + - ((namespaces && namespaces !== '') ? '&namespaces=' + namespaces : '') + - '&option=sort(%2BthingId)' + - // ',size(3)' + - (cursor ? ',cursor(' + cursor + ')' : ''), - ).then((searchResult) => { - if (cursor) { - removeMoreFromThingList(); - } else { - theSearchCursor = null; - dom.thingsTableBody.innerHTML = ''; - } - fillThingsTable(searchResult.items); - checkMorePages(searchResult); - }).catch((error) => { - theSearchCursor = null; - dom.thingsTableBody.innerHTML = ''; - }).finally(() => { - document.body.style.cursor = 'default'; - }); -} - -/** - * Gets things from Ditto by thingIds and fills the UI with the result - * @param {Array} thingIds Array of thingIds - */ -export function getThings(thingIds) { - dom.thingsTableBody.innerHTML = ''; - if (thingIds.length > 0) { - API.callDittoREST('GET', - `/things?${Fields.getQueryParameter()}&ids=${thingIds}&option=sort(%2BthingId)`, - ).then(fillThingsTable); - } -} - -/** - * Returns a click handler for Update thing and delete thing - * @param {String} method PUT or DELETE - */ -function modifyThing(method) { - API.callDittoREST(method, - '/things/' + dom.thingId.value, - method === 'PUT' ? JSON.parse(thingJsonEditor.getValue()) : null, - { - 'if-match': '*', - }, - ).then(() => { - method === 'PUT' ? refreshThing(dom.thingId.value) : SearchFilter.performLastSearch(); - }); } /** @@ -297,90 +66,9 @@ export function refreshThing(thingId, successCallback) { * Update all UI components for the given Thing * @param {Object} thingJson Thing json */ -function setTheThing(thingJson) { +export function setTheThing(thingJson) { theThing = thingJson; - - updateThingDetailsTable(); - updateThingJsonEditor(); - observers.forEach((observer) => observer.call(null, theThing)); - - function updateThingDetailsTable() { - dom.thingDetails.innerHTML = ''; - if (theThing) { - Utils.addTableRow(dom.thingDetails, 'thingId', false, true, theThing.thingId); - Utils.addTableRow(dom.thingDetails, 'policyId', false, true, theThing.policyId); - Utils.addTableRow(dom.thingDetails, 'definition', false, true, theThing.definition ?? ''); - Utils.addTableRow(dom.thingDetails, 'revision', false, true, theThing._revision); - Utils.addTableRow(dom.thingDetails, 'created', false, true, theThing._created); - Utils.addTableRow(dom.thingDetails, 'modified', false, true, theThing._modified); - } - } - - function updateThingJsonEditor() { - if (theThing) { - dom.thingId.value = theThing.thingId; - dom.inputThingDefinition.value = theThing.definition ?? ''; - const thingCopy = JSON.parse(JSON.stringify(theThing)); - delete thingCopy['_revision']; - delete thingCopy['_created']; - delete thingCopy['_modified']; - thingJsonEditor.setValue(JSON.stringify(thingCopy, null, 2), -1); - } else { - dom.thingId.value = null; - dom.inputThingDefinition.value = null; - thingJsonEditor.setValue(''); - } - } -} - -/** - * Updates UI depepending on existing additional pages on Ditto things search - * @param {Object} searchResult Result from Ditto thing search - */ -function checkMorePages(searchResult) { - if (searchResult['cursor']) { - addMoreToThingList(); - theSearchCursor = searchResult.cursor; - } else { - theSearchCursor = null; - } -} - -/** - * Adds a clickable "more" line to the things table UI - */ -function addMoreToThingList() { - const moreCell = dom.thingsTableBody.insertRow().insertCell(-1); - moreCell.innerHTML = 'load more...'; - moreCell.colSpan = dom.thingsTableBody.rows[0].childElementCount; - moreCell.style.textAlign = 'center'; - moreCell.style.cursor = 'pointer'; - moreCell.disabled = true; - moreCell.style.color = '#3a8c9a'; - moreCell.parentNode.id = 'searchThingsMore'; -} - -/** - * remove the "more" line from the things table - */ -function removeMoreFromThingList() { - const moreRow = document.getElementById('searchThingsMore'); - if (moreRow) { - moreRow.parentNode.removeChild(moreRow); - } -} - -function togglePinnedThing(evt) { - if (evt.target.checked) { - Environments.current().pinnedThings.push(this.id); - } else { - const index = Environments.current().pinnedThings.indexOf(this.id); - if (index > -1) { - Environments.current().pinnedThings.splice(index, 1); - } - } - Environments.environmentsJsonChanged('pinnedThings'); } let viewDirty = false; @@ -404,7 +92,7 @@ function onEnvironmentChanged(modifiedField) { } function refreshView() { - SearchFilter.performLastSearch(); + ThingsSearch.performLastSearch(); } diff --git a/ui/modules/things/thingsCRUD.js b/ui/modules/things/thingsCRUD.js new file mode 100644 index 00000000000..faa55f821af --- /dev/null +++ b/ui/modules/things/thingsCRUD.js @@ -0,0 +1,168 @@ +/* +* Copyright (c) 2022 Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Eclipse Public License 2.0 which is available at +* http://www.eclipse.org/legal/epl-2.0 +* +* SPDX-License-Identifier: EPL-2.0 +*/ +/* eslint-disable require-jsdoc */ +// @ts-check +import * as API from '../api.js'; + +import * as Utils from '../utils.js'; +import * as ThingsSearch from './thingsSearch.js'; +import * as Things from './things.js'; + +let thingJsonEditor; + +let thingTemplates; + +const dom = { + thingDetails: null, + thingId: null, + buttonCreateThing: null, + buttonSaveThing: null, + buttonDeleteThing: null, + inputThingDefinition: null, + ulThingDefinitions: null, +}; + +/** + * Initializes components. Should be called after DOMContentLoaded event + */ +export async function ready() { + Things.addChangeListener(onThingChanged); + + Utils.getAllElementsById(dom); + + thingJsonEditor = Utils.createAceEditor('thingJsonEditor', 'ace/mode/json'); + + loadThingTemplates(); + + dom.ulThingDefinitions.addEventListener('click', onThingDefinitionsClick); + dom.buttonCreateThing.onclick = onCreateThingClick; + dom.buttonSaveThing.onclick = onSaveThingClick; + dom.buttonDeleteThing.onclick = onDeleteThingClick; + + document.querySelector('a[data-bs-target="#tabModifyThing"]').addEventListener('shown.bs.tab', (event) => { + thingJsonEditor.renderer.updateFull(); + }); +} + +function onDeleteThingClick() { + return () => { + Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId); + Utils.confirm(`Are you sure you want to delete thing
'${dom.thingId.value}'?`, 'Delete', () => { + deleteThing(dom.thingId.value); + }); + }; +} + +function onSaveThingClick() { + Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId); + putThing(dom.thingId.value, JSON.parse(thingJsonEditor.getValue())); +} + +async function onCreateThingClick() { + const editorValue = thingJsonEditor.getValue(); + if (dom.thingId.value !== undefined && dom.thingId.value !== '') { + API.callDittoREST('PUT', + '/things/' + dom.thingId.value, + editorValue === '' ? {} : JSON.parse(editorValue), + { + 'if-none-match': '*', + }, + ).then((data) => { + Things.refreshThing(data.thingId, () => { + ThingsSearch.getThings([data.thingId]); + }); + }); + } else { + API.callDittoREST('POST', '/things', editorValue === '' ? {} : JSON.parse(editorValue)) + .then((data) => { + Things.refreshThing(data.thingId, () => { + ThingsSearch.getThings([data.thingId]); + }); + }); + } +} + +function onThingDefinitionsClick(event) { + Things.setTheThing(null); + dom.inputThingDefinition.value = event.target.textContent; + thingJsonEditor.setValue(JSON.stringify(thingTemplates[event.target.textContent], null, 2), -1); +} + +function loadThingTemplates() { + fetch('templates/thingTemplates.json') + .then((response) => { + response.json().then((loadedTemplates) => { + thingTemplates = loadedTemplates; + Utils.addDropDownEntries(dom.ulThingDefinitions, Object.keys(thingTemplates)); + }); + }); +} + +function putThing(thingId, thingJson) { + API.callDittoREST('PUT', `/things/${thingId}`, thingJson, + { + 'if-match': '*', + }, + ).then(() => { + Things.refreshThing(thingId, null); + }); +} + +function deleteThing(thingId) { + API.callDittoREST('DELETE', `/things/${thingId}`, null, + { + 'if-match': '*', + }, + ).then(() => { + ThingsSearch.performLastSearch(); + }); +} + +/** + * Update UI components for the given Thing + * @param {Object} thingJson Thing json + */ +function onThingChanged(thingJson) { + updateThingDetailsTable(); + updateThingJsonEditor(); + + function updateThingDetailsTable() { + dom.thingDetails.innerHTML = ''; + if (thingJson) { + Utils.addTableRow(dom.thingDetails, 'thingId', false, true, thingJson.thingId); + Utils.addTableRow(dom.thingDetails, 'policyId', false, true, thingJson.policyId); + Utils.addTableRow(dom.thingDetails, 'definition', false, true, thingJson.definition ?? ''); + Utils.addTableRow(dom.thingDetails, 'revision', false, true, thingJson._revision); + Utils.addTableRow(dom.thingDetails, 'created', false, true, thingJson._created); + Utils.addTableRow(dom.thingDetails, 'modified', false, true, thingJson._modified); + } + } + + function updateThingJsonEditor() { + if (thingJson) { + dom.thingId.value = thingJson.thingId; + dom.inputThingDefinition.value = thingJson.definition ?? ''; + const thingCopy = JSON.parse(JSON.stringify(thingJson)); + delete thingCopy['_revision']; + delete thingCopy['_created']; + delete thingCopy['_modified']; + thingJsonEditor.setValue(JSON.stringify(thingCopy, null, 2), -1); + } else { + dom.thingId.value = null; + dom.inputThingDefinition.value = null; + thingJsonEditor.setValue(''); + } + } +} + + diff --git a/ui/modules/things/thingsSearch.js b/ui/modules/things/thingsSearch.js new file mode 100644 index 00000000000..f8a9bce4a76 --- /dev/null +++ b/ui/modules/things/thingsSearch.js @@ -0,0 +1,260 @@ +/* +* Copyright (c) 2022 Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Eclipse Public License 2.0 which is available at +* http://www.eclipse.org/legal/epl-2.0 +* +* SPDX-License-Identifier: EPL-2.0 +*/ + +/* eslint-disable require-jsdoc */ +/* eslint-disable new-cap */ +/* eslint-disable no-invalid-this */ + +// @ts-check + +// @ts-ignore +import {JSONPath} from 'https://cdn.jsdelivr.net/npm/jsonpath-plus@5.0.3/dist/index-browser-esm.min.js'; + +import * as API from '../api.js'; + +import * as Utils from '../utils.js'; +import * as Fields from './fields.js'; +import * as Things from './things.js'; +import * as Environments from '../environments/environments.js'; + +let lastSearch = ''; +let theSearchCursor; + +const dom = { + thingsTableHead: null, + thingsTableBody: null, + searchFilterEdit: null, + favIcon: null, +}; + +export async function ready() { + Things.addChangeListener(onThingChanged); + + Utils.getAllElementsById(dom); + + dom.thingsTableBody.addEventListener('click', onThingsTableClicked); +} + +function onThingsTableClicked(event) { + if (event.target && event.target.nodeName === 'TD') { + const row = event.target.parentNode; + if (row.id === 'searchThingsMore') { + row.style.pointerEvents = 'none'; + event.stopImmediatePropagation(); + searchThings(dom.searchFilterEdit.value, true); + } else { + if (Things.theThing && Things.theThing.thingId === row.id) { + Things.setTheThing(null); + } else { + Things.refreshThing(row.id, null); + } + } + } +} + +/** + * Tests if the search filter is an RQL. If yes, things search is called otherwise just things get + * @param {String} filter search filter string containing an RQL or a thingId + */ +export function searchTriggered(filter) { + lastSearch = filter; + const regex = /^(eq\(|ne\(|gt\(|ge\(|lt\(|le\(|in\(|like\(|exists\(|and\(|or\(|not\().*/; + if (filter === '' || regex.test(filter)) { + searchThings(filter); + } else { + getThings([filter]); + } +} + +/** + * Gets the list of pinned things + */ +export function pinnedTriggered() { + lastSearch = 'pinned'; + dom.searchFilterEdit.value = null; + dom.favIcon.classList.replace('bi-star-fill', 'bi-star'); + getThings(Environments.current()['pinnedThings']); +} + +/** + * Performs the last search by the user using the last used filter. + * If the user used pinned things last time, the pinned things are reloaded + */ +export function performLastSearch() { + if (lastSearch === 'pinned') { + pinnedTriggered(); + } else { + searchTriggered(lastSearch); + } +} + +/** + * Gets things from Ditto by thingIds and fills the UI with the result + * @param {Array} thingIds Array of thingIds + */ +export function getThings(thingIds) { + dom.thingsTableBody.innerHTML = ''; + if (thingIds.length > 0) { + API.callDittoREST('GET', + `/things?${Fields.getQueryParameter()}&ids=${thingIds}&option=sort(%2BthingId)`, + ).then(fillThingsTable); + } +} + +/** + * Calls Ditto search api and fills UI with the result + * @param {String} filter Ditto search filter (rql) + * @param {boolean} isMore (optional) use cursor from previous search for additional pages + */ +export function searchThings(filter, isMore = false) { + document.body.style.cursor = 'progress'; + + const namespaces = Environments.current().searchNamespaces; + + API.callDittoREST('GET', + '/search/things?' + Fields.getQueryParameter() + + ((filter && filter !== '') ? '&filter=' + encodeURIComponent(filter) : '') + + ((namespaces && namespaces !== '') ? '&namespaces=' + namespaces : '') + + '&option=sort(%2BthingId)' + + // ',size(3)' + + (isMore ? ',cursor(' + theSearchCursor + ')' : ''), + ).then((searchResult) => { + if (isMore) { + removeMoreFromThingList(); + } else { + theSearchCursor = null; + dom.thingsTableBody.innerHTML = ''; + } + fillThingsTable(searchResult.items); + checkMorePages(searchResult); + }).catch((error) => { + theSearchCursor = null; + dom.thingsTableBody.innerHTML = ''; + }).finally(() => { + document.body.style.cursor = 'default'; + }); + + function checkMorePages(searchResult) { + if (searchResult['cursor']) { + addMoreToThingList(); + theSearchCursor = searchResult.cursor; + } else { + theSearchCursor = null; + } + } + + function addMoreToThingList() { + const moreCell = dom.thingsTableBody.insertRow().insertCell(-1); + moreCell.innerHTML = 'load more...'; + moreCell.colSpan = dom.thingsTableBody.rows[0].childElementCount; + moreCell.style.textAlign = 'center'; + moreCell.style.cursor = 'pointer'; + moreCell.disabled = true; + moreCell.style.color = '#3a8c9a'; + moreCell.parentNode.id = 'searchThingsMore'; + } +} + +/** + * remove the "more" line from the things table + */ +export function removeMoreFromThingList() { + const moreRow = document.getElementById('searchThingsMore'); + if (moreRow) { + moreRow.parentNode.removeChild(moreRow); + } +} + + +/** + * Fills the things table UI with the given things + * @param {Array} thingsList Array of thing json objects + */ +function fillThingsTable(thingsList) { + const activeFields = Environments.current().fieldList.filter((f) => f.active); + fillHeaderRow(); + let thingSelected = false; + thingsList.forEach((item, t) => { + const row = dom.thingsTableBody.insertRow(); + fillBodyRow(row, item); + }); + if (!thingSelected) { + Things.setTheThing(null); + } + + function fillHeaderRow() { + dom.thingsTableHead.innerHTML = ''; + // Utils.addCheckboxToRow(dom.thingsTableHead, 'checkboxHead', false, null); + Utils.insertHeaderCell(dom.thingsTableHead, ''); + Utils.insertHeaderCell(dom.thingsTableHead, 'Thing ID'); + activeFields.forEach((field) => { + Utils.insertHeaderCell(dom.thingsTableHead, field['label'] ? field.label : field.path); + }); + } + + function fillBodyRow(row, item) { + row.id = item.thingId; + if (Things.theThing && (item.thingId === Things.theThing.thingId)) { + thingSelected = true; + row.classList.add('table-active'); + } + Utils.addCheckboxToRow( + row, + item.thingId, + Environments.current().pinnedThings.includes(item.thingId), + togglePinnedThing, + ); + Utils.addCellToRow(row, beautifyId(item.thingId), item.thingId); + activeFields.forEach((field) => { + let path = field.path.replace(/\//g, '.'); + if (path.charAt(0) !== '.') { + path = '$.' + path; + } + const elem = JSONPath({ + json: item, + path: path, + }); + Utils.addCellToRow(row, elem.length !== 0 ? elem[0] : ''); + }); + } + + function beautifyId(longId) { + const uuidRegex = /([0-9a-f]{7})[0-9a-f]-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g; + let result = longId; + if (Environments.current()['shortenUUID']) { + result = result.replace(uuidRegex, '$1'); + } + if (Environments.current()['defaultNamespace']) { + result = result.replace(Environments.current()['defaultNamespace'], 'dn'); + } + return result; + } +} + +function togglePinnedThing(evt) { + if (evt.target.checked) { + Environments.current().pinnedThings.push(this.id); + } else { + const index = Environments.current().pinnedThings.indexOf(this.id); + if (index > -1) { + Environments.current().pinnedThings.splice(index, 1); + } + } + Environments.environmentsJsonChanged('pinnedThings'); +} + +function onThingChanged(thingJson) { + if (!thingJson) { + Utils.tableAdjustSelection(dom.thingsTableBody, () => false); + } +} From 4d09b9013338db1b777f08bc3f6b5473ad48f4fe Mon Sep 17 00:00:00 2001 From: thfries Date: Mon, 7 Nov 2022 06:23:56 +0100 Subject: [PATCH 002/173] Add SSE support to Explorer UI Signed-off-by: thfries --- ui/index.html | 6 ++++++ ui/modules/api.js | 10 ++++++++++ ui/modules/things/things.js | 21 +++++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/ui/index.html b/ui/index.html index 4778015816b..a0ac85ed67c 100644 --- a/ui/index.html +++ b/ui/index.html @@ -32,6 +32,12 @@ + + diff --git a/ui/modules/api.js b/ui/modules/api.js index 67a89c7663f..83044077291 100644 --- a/ui/modules/api.js +++ b/ui/modules/api.js @@ -333,6 +333,16 @@ export async function callDittoREST(method, path, body, additionalHeaders) { } } +export function getEventSource(thingId) { + return new EventSourcePolyfill( + Environments.current().api_uri + '/api/2/things?ids=' + thingId, { + headers: { + [authHeaderKey]: authHeaderValue, + }, + }, + ); +} + /** * Calls connections api. Uses devops api in case of Ditto and the solutions api in case of Things * @param {*} operation connections api operation diff --git a/ui/modules/things/things.js b/ui/modules/things/things.js index e8fff1e728c..e099fe1cd04 100644 --- a/ui/modules/things/things.js +++ b/ui/modules/things/things.js @@ -20,6 +20,8 @@ import * as ThingsSearch from './thingsSearch.js'; export let theThing; +let thingEventSource; + const observers = []; const dom = { @@ -67,10 +69,29 @@ export function refreshThing(thingId, successCallback) { * @param {Object} thingJson Thing json */ export function setTheThing(thingJson) { + adjustEventSource(thingJson); theThing = thingJson; observers.forEach((observer) => observer.call(null, theThing)); } +function adjustEventSource(newThingJson) { + if (!newThingJson) { + thingEventSource && thingEventSource.close(); + } else { + if (!theThing || theThing.thingId !== newThingJson.thingId) { + console.log('Start SSE: ' + newThingJson.thingId); + thingEventSource = API.getEventSource(newThingJson.thingId); + thingEventSource.onmessage = (event) => { + console.log(event); + if (event.data && event.data !== '') { + const merged = _.merge(theThing, JSON.parse(event.data)); + setTheThing(merged); + } + }; + } + } +} + let viewDirty = false; function onTabActivated() { From 7f7c44e4ab6d1b3575e38c3a1271463dd82ab846 Mon Sep 17 00:00:00 2001 From: thfries Date: Mon, 19 Dec 2022 07:48:58 +0100 Subject: [PATCH 003/173] SSE for explorer UI next step - Explicit editing for thing CRUD - Split of thing.js file Signed-off-by: thfries --- ui/index.css | 19 +++++++ ui/index.html | 1 + ui/main.js | 2 + ui/modules/api.js | 4 +- ui/modules/things/things.html | 23 +++++--- ui/modules/things/things.js | 26 +-------- ui/modules/things/thingsCRUD.js | 92 ++++++++++++++++++++++--------- ui/modules/things/thingsSSE.js | 48 ++++++++++++++++ ui/modules/things/thingsSearch.js | 1 + 9 files changed, 155 insertions(+), 61 deletions(-) create mode 100644 ui/modules/things/thingsSSE.js diff --git a/ui/index.css b/ui/index.css index 21689489a4e..42d6adc8627 100644 --- a/ui/index.css +++ b/ui/index.css @@ -151,3 +151,22 @@ h5, h6 { hr { margin: 0.25rem; } + +.editBackground { + display: block; + position: fixed; + z-index: 1100; + left: 0; + top: 0; + width: 100%; + height: 100%; + overflow: auto; + background-color: rgb(0,0,0); + background-color: rgba(0,0,0,0.4); + } + + .editForground { + z-index: 1200; + position: relative; + background-color: white; + } diff --git a/ui/index.html b/ui/index.html index a0ac85ed67c..cc3d1f59465 100644 --- a/ui/index.html +++ b/ui/index.html @@ -128,6 +128,7 @@ +
diff --git a/ui/main.js b/ui/main.js index 25855cec186..cddd2a77f83 100644 --- a/ui/main.js +++ b/ui/main.js @@ -22,6 +22,7 @@ import * as SearchFilter from './modules/things/searchFilter.js'; import * as Things from './modules/things/things.js'; import * as ThingsSearch from './modules/things/thingsSearch.js'; import * as ThingsCRUD from './modules/things/thingsCRUD.js'; +import * as ThingsSSE from './modules/things/thingsSSE.js'; import * as Connections from './modules/connections/connections.js'; import * as Policies from './modules/policies/policies.js'; import * as API from './modules/api.js'; @@ -48,6 +49,7 @@ document.addEventListener('DOMContentLoaded', async function() { await Things.ready(); ThingsSearch.ready(); ThingsCRUD.ready(); + ThingsSSE.ready(); Attributes.ready(); await Fields.ready(); await SearchFilter.ready(); diff --git a/ui/modules/api.js b/ui/modules/api.js index 83044077291..a512d2c746e 100644 --- a/ui/modules/api.js +++ b/ui/modules/api.js @@ -333,9 +333,9 @@ export async function callDittoREST(method, path, body, additionalHeaders) { } } -export function getEventSource(thingId) { +export function getEventSource(thingId, urlParams) { return new EventSourcePolyfill( - Environments.current().api_uri + '/api/2/things?ids=' + thingId, { + `${Environments.current().api_uri}/api/2/things?ids=${thingId}${urlParams ?? ''}`, { headers: { [authHeaderKey]: authHeaderValue, }, diff --git a/ui/modules/things/things.html b/ui/modules/things/things.html index c1cca4278fc..67a85438ee4 100644 --- a/ui/modules/things/things.html +++ b/ui/modules/things/things.html @@ -59,7 +59,8 @@
Things
Details
@@ -71,18 +72,22 @@
Things
-
+
- - + - -
@@ -91,12 +96,12 @@
Things
+ data-bs-toggle="dropdown" disabled id="buttonThingDefinitions">
- +
-
+
diff --git a/ui/modules/things/things.js b/ui/modules/things/things.js index e099fe1cd04..2c0aaa2a487 100644 --- a/ui/modules/things/things.js +++ b/ui/modules/things/things.js @@ -20,8 +20,6 @@ import * as ThingsSearch from './thingsSearch.js'; export let theThing; -let thingEventSource; - const observers = []; const dom = { @@ -54,6 +52,7 @@ export async function ready() { * @param {function} successCallback callback function that is called after refresh is finished */ export function refreshThing(thingId, successCallback) { + console.assert(thingId && thingId !== '', 'thingId expected'); API.callDittoREST('GET', `/things/${thingId}?` + 'fields=thingId%2CpolicyId%2Cdefinition%2Cattributes%2Cfeatures%2C_created%2C_modified%2C_revision') @@ -69,27 +68,9 @@ export function refreshThing(thingId, successCallback) { * @param {Object} thingJson Thing json */ export function setTheThing(thingJson) { - adjustEventSource(thingJson); + const isNewThingId = thingJson && (!theThing || theThing.thingId !== thingJson.thingId); theThing = thingJson; - observers.forEach((observer) => observer.call(null, theThing)); -} - -function adjustEventSource(newThingJson) { - if (!newThingJson) { - thingEventSource && thingEventSource.close(); - } else { - if (!theThing || theThing.thingId !== newThingJson.thingId) { - console.log('Start SSE: ' + newThingJson.thingId); - thingEventSource = API.getEventSource(newThingJson.thingId); - thingEventSource.onmessage = (event) => { - console.log(event); - if (event.data && event.data !== '') { - const merged = _.merge(theThing, JSON.parse(event.data)); - setTheThing(merged); - } - }; - } - } + observers.forEach((observer) => observer.call(null, theThing, isNewThingId)); } let viewDirty = false; @@ -116,4 +97,3 @@ function refreshView() { ThingsSearch.performLastSearch(); } - diff --git a/ui/modules/things/thingsCRUD.js b/ui/modules/things/thingsCRUD.js index faa55f821af..90e613ef323 100644 --- a/ui/modules/things/thingsCRUD.js +++ b/ui/modules/things/thingsCRUD.js @@ -20,14 +20,21 @@ import * as Things from './things.js'; let thingJsonEditor; +let isEditing = false; + let thingTemplates; const dom = { thingDetails: null, thingId: null, + modalThingsEdit: null, + iThingsEdit: null, + divThingsCRUD: null, + buttonEditThing: null, buttonCreateThing: null, buttonSaveThing: null, buttonDeleteThing: null, + buttonThingDefinitions: null, inputThingDefinition: null, ulThingDefinitions: null, }; @@ -40,7 +47,7 @@ export async function ready() { Utils.getAllElementsById(dom); - thingJsonEditor = Utils.createAceEditor('thingJsonEditor', 'ace/mode/json'); + thingJsonEditor = Utils.createAceEditor('thingJsonEditor', 'ace/mode/json', true); loadThingTemplates(); @@ -48,6 +55,7 @@ export async function ready() { dom.buttonCreateThing.onclick = onCreateThingClick; dom.buttonSaveThing.onclick = onSaveThingClick; dom.buttonDeleteThing.onclick = onDeleteThingClick; + dom.buttonEditThing.onclick = toggleEdit; document.querySelector('a[data-bs-target="#tabModifyThing"]').addEventListener('shown.bs.tab', (event) => { thingJsonEditor.renderer.updateFull(); @@ -55,17 +63,28 @@ export async function ready() { } function onDeleteThingClick() { - return () => { - Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId); - Utils.confirm(`Are you sure you want to delete thing
'${dom.thingId.value}'?`, 'Delete', () => { - deleteThing(dom.thingId.value); + Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId); + Utils.confirm(`Are you sure you want to delete thing
'${dom.thingId.value}'?`, 'Delete', () => { + API.callDittoREST('DELETE', `/things/${dom.thingId.value}`, null, + { + 'if-match': '*', + }, + ).then(() => { + ThingsSearch.performLastSearch(); }); - }; + }); } function onSaveThingClick() { Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId); - putThing(dom.thingId.value, JSON.parse(thingJsonEditor.getValue())); + API.callDittoREST('PUT', `/things/${dom.thingId.value}`, JSON.parse(thingJsonEditor.getValue()), + { + 'if-match': '*', + }, + ).then(() => { + toggleEdit(); + Things.refreshThing(dom.thingId.value, null); + }); } async function onCreateThingClick() { @@ -78,6 +97,7 @@ async function onCreateThingClick() { 'if-none-match': '*', }, ).then((data) => { + toggleEdit(); Things.refreshThing(data.thingId, () => { ThingsSearch.getThings([data.thingId]); }); @@ -85,6 +105,7 @@ async function onCreateThingClick() { } else { API.callDittoREST('POST', '/things', editorValue === '' ? {} : JSON.parse(editorValue)) .then((data) => { + toggleEdit(); Things.refreshThing(data.thingId, () => { ThingsSearch.getThings([data.thingId]); }); @@ -94,6 +115,7 @@ async function onCreateThingClick() { function onThingDefinitionsClick(event) { Things.setTheThing(null); + isEditing = true; dom.inputThingDefinition.value = event.target.textContent; thingJsonEditor.setValue(JSON.stringify(thingTemplates[event.target.textContent], null, 2), -1); } @@ -108,31 +130,15 @@ function loadThingTemplates() { }); } -function putThing(thingId, thingJson) { - API.callDittoREST('PUT', `/things/${thingId}`, thingJson, - { - 'if-match': '*', - }, - ).then(() => { - Things.refreshThing(thingId, null); - }); -} - -function deleteThing(thingId) { - API.callDittoREST('DELETE', `/things/${thingId}`, null, - { - 'if-match': '*', - }, - ).then(() => { - ThingsSearch.performLastSearch(); - }); -} - /** * Update UI components for the given Thing * @param {Object} thingJson Thing json */ function onThingChanged(thingJson) { + if (isEditing) { + return; + } + updateThingDetailsTable(); updateThingJsonEditor(); @@ -151,6 +157,7 @@ function onThingChanged(thingJson) { function updateThingJsonEditor() { if (thingJson) { dom.thingId.value = thingJson.thingId; + dom.buttonDeleteThing.disabled = false; dom.inputThingDefinition.value = thingJson.definition ?? ''; const thingCopy = JSON.parse(JSON.stringify(thingJson)); delete thingCopy['_revision']; @@ -159,6 +166,37 @@ function onThingChanged(thingJson) { thingJsonEditor.setValue(JSON.stringify(thingCopy, null, 2), -1); } else { dom.thingId.value = null; + dom.buttonDeleteThing.disabled = true; + dom.inputThingDefinition.value = null; + thingJsonEditor.setValue(''); + } + } +} + +function toggleEdit() { + isEditing = !isEditing; + dom.modalThingsEdit.classList.toggle('editBackground'); + dom.divThingsCRUD.classList.toggle('editForground'); + dom.iThingsEdit.classList.toggle('bi-pencil-square'); + dom.iThingsEdit.classList.toggle('bi-x-square'); + dom.buttonThingDefinitions.disabled = !dom.buttonThingDefinitions.disabled; + dom.inputThingDefinition.disabled = !dom.inputThingDefinition.disabled; + thingJsonEditor.setReadOnly(!isEditing); + thingJsonEditor.renderer.setShowGutter(isEditing); + if (dom.thingId.value) { + dom.buttonSaveThing.disabled = !dom.buttonSaveThing.disabled; + } else { + dom.buttonCreateThing.disabled = !dom.buttonCreateThing.disabled; + dom.thingId.disabled = !dom.thingId.disabled; + } + if (!isEditing) { + clearEditorsAfterCancel(); + } + + function clearEditorsAfterCancel() { + if (dom.thingId.value && dom.thingId.value !== '') { + Things.refreshThing(dom.thingId.value, null); + } else { dom.inputThingDefinition.value = null; thingJsonEditor.setValue(''); } diff --git a/ui/modules/things/thingsSSE.js b/ui/modules/things/thingsSSE.js new file mode 100644 index 00000000000..f87eb7a7039 --- /dev/null +++ b/ui/modules/things/thingsSSE.js @@ -0,0 +1,48 @@ +/* +* Copyright (c) 2022 Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Eclipse Public License 2.0 which is available at +* http://www.eclipse.org/legal/epl-2.0 +* +* SPDX-License-Identifier: EPL-2.0 +*/ +/* eslint-disable require-jsdoc */ +// @ts-check +import * as API from '../api.js'; + +import * as Things from './things.js'; + +let thingEventSource; + + +/** + * Initializes components. Should be called after DOMContentLoaded event + */ +export async function ready() { + Things.addChangeListener(onThingChanged); +} + +function onThingChanged(newThingJson, isNewThingId) { + if (!newThingJson) { + thingEventSource && thingEventSource.close(); + thingEventSource = null; + } else if (isNewThingId) { + thingEventSource && thingEventSource.close(); + console.log('Start SSE: ' + newThingJson.thingId); + thingEventSource = API.getEventSource(newThingJson.thingId, '&extraFields=_revision,_modified'); + thingEventSource.onmessage = onMessage; + } +} + +function onMessage(event) { + if (event.data && event.data !== '') { + console.log(event); + const merged = _.merge(Things.theThing, JSON.parse(event.data)); + Things.setTheThing(merged); + } +} + diff --git a/ui/modules/things/thingsSearch.js b/ui/modules/things/thingsSearch.js index f8a9bce4a76..1eb7674bf6f 100644 --- a/ui/modules/things/thingsSearch.js +++ b/ui/modules/things/thingsSearch.js @@ -54,6 +54,7 @@ function onThingsTableClicked(event) { searchThings(dom.searchFilterEdit.value, true); } else { if (Things.theThing && Things.theThing.thingId === row.id) { + event.stopImmediatePropagation(); Things.setTheThing(null); } else { Things.refreshThing(row.id, null); From 0a6e4113278f7915d5b672912ff2e292463f3e87 Mon Sep 17 00:00:00 2001 From: thfries Date: Mon, 19 Dec 2022 20:00:21 +0100 Subject: [PATCH 004/173] SSE for explorer UI: resolved error from merge Signed-off-by: thfries --- ui/modules/things/searchFilter.js | 44 +++---------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/ui/modules/things/searchFilter.js b/ui/modules/things/searchFilter.js index 55cd00d0eda..c0a97d54e6b 100644 --- a/ui/modules/things/searchFilter.js +++ b/ui/modules/things/searchFilter.js @@ -1,3 +1,4 @@ +/* eslint-disable require-jsdoc */ /* * Copyright (c) 2022 Contributors to the Eclipse Foundation * @@ -30,8 +31,6 @@ const filterHistory = []; let keyStrokeTimeout; -let lastSearch = ''; - const FILTER_PLACEHOLDER = '*****'; const dom = { @@ -61,12 +60,13 @@ export async function ready() { checkIfFavourite(); const filterEditNeeded = checkAndMarkParameter(); if (!filterEditNeeded) { - Things.searchThings(event.target.textContent); + ThingsSearch.searchThings(event.target.textContent); } } }); dom.searchThings.onclick = () => { + fillHistory(dom.searchFilterEdit.value); ThingsSearch.searchTriggered(dom.searchFilterEdit.value); }; @@ -79,6 +79,7 @@ export async function ready() { dom.searchFilterEdit.onkeyup = (event) => { if (event.key === 'Enter' || event.code === 13) { + fillHistory(dom.searchFilterEdit.value); ThingsSearch.searchTriggered(dom.searchFilterEdit.value); } else { clearTimeout(keyStrokeTimeout); @@ -110,43 +111,6 @@ function onEnvironmentChanged() { updateFilterList(); } -/** - * Tests if the search filter is an RQL. If yes, things search is called otherwise just things get - * @param {String} filter search filter string containing an RQL or a thingId - */ -function searchTriggered(filter) { - lastSearch = filter; - fillHistory(filter); - const regex = /^(eq\(|ne\(|gt\(|ge\(|lt\(|le\(|in\(|like\(|exists\(|and\(|or\(|not\().*/; - if (filter === '' || regex.test(filter)) { - Things.searchThings(filter); - } else { - Things.getThings([filter]); - } -} - -/** - * Gets the list of pinned things - */ -function pinnedTriggered() { - lastSearch = 'pinned'; - dom.searchFilterEdit.value = null; - dom.favIcon.classList.replace('bi-star-fill', 'bi-star'); - Things.getThings(Environments.current()['pinnedThings']); -} - -/** - * Performs the last search by the user using the last used filter. - * If the user used pinned things last time, the pinned things are reloaded - */ -export function performLastSearch() { - if (lastSearch === 'pinned') { - pinnedTriggered(); - } else { - searchTriggered(lastSearch); - } -} - /** * Updates the UI filterList */ From 01c921eb719e4878bd301bca27473b3e46c58a54 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Mon, 26 Sep 2022 15:07:43 +0200 Subject: [PATCH 005/173] Enable "History API" support: * streaming of journal entries of a given entity (thing/policy) as DittoProtocol events (via Websocket/Connections/SSE) * accessing an entity (thing/policy/connection) "at" a specific revision or "at" a specific timestamp * added a configurable "history-retention-duration" config for things/policies/connections in order to delay the cleanup for at least that amount of time -> that way we can e.g. keep 30d of snapshots/events * made history API work via connectivity * added documentation for history capabilities Signed-off-by: Thomas Jaeckle --- .../CleanupPersistenceResponseTest.java | 3 +- .../cleanup/CleanupPersistenceTest.java | 3 +- .../ditto/base/model/entity/id/EntityId.java | 8 +- .../model/headers/DittoHeaderDefinition.java | 40 +- .../base/model/signals/FeatureToggle.java | 42 +++ .../signals/WithStreamingSubscriptionId.java | 50 +++ .../base/model/signals/commands/Command.java | 8 +- ...treamingSubscriptionNotFoundException.java | 110 ++++++ ...ingSubscriptionProtocolErrorException.java | 121 ++++++ ...StreamingSubscriptionTimeoutException.java | 110 ++++++ .../AbstractStreamingSubscriptionCommand.java | 140 +++++++ .../CancelStreamingSubscription.java | 146 ++++++++ .../RequestFromStreamingSubscription.java | 188 ++++++++++ .../StreamingSubscriptionCommand.java | 73 ++++ .../SubscribeForPersistedEvents.java | 344 ++++++++++++++++++ .../commands/streaming/package-info.java | 15 + .../model/signals/events/AbstractEvent.java | 1 + .../events/AbstractEventsourcedEvent.java | 1 + .../signals/events/EventJsonDeserializer.java | 4 +- .../AbstractStreamingSubscriptionEvent.java | 172 +++++++++ .../StreamingSubscriptionComplete.java | 106 ++++++ .../StreamingSubscriptionCreated.java | 108 ++++++ .../streaming/StreamingSubscriptionEvent.java | 76 ++++ .../StreamingSubscriptionFailed.java | 166 +++++++++ .../StreamingSubscriptionHasNext.java | 165 +++++++++ .../events/streaming/package-info.java | 15 + .../headers/ImmutableDittoHeadersTest.java | 19 +- .../signals/ShardedMessageEnvelopeTest.java | 7 +- .../CancelStreamingSubscriptionTest.java | 57 +++ .../RequestFromStreamingSubscriptionTest.java | 56 +++ .../SubscribeForPersistedEventsTest.java | 136 +++++++ .../StreamingSubscriptionCompleteTest.java | 51 +++ .../StreamingSubscriptionCreatedTest.java | 51 +++ .../StreamingSubscriptionFailedTest.java | 58 +++ .../StreamingSubscriptionHasNextTest.java | 56 +++ ...nnectionHistoryNotAccessibleException.java | 178 +++++++++ .../service/ConnectivityRootActor.java | 6 +- .../service/config/ConnectionConfig.java | 12 +- .../config/DefaultConnectionConfig.java | 13 +- .../service/messaging/BaseClientActor.java | 56 ++- .../service/messaging/BasePublisherActor.java | 11 +- .../messaging/InboundDispatchingSink.java | 6 +- .../messaging/OutboundDispatchingActor.java | 2 + .../ConnectionPersistenceActor.java | 46 ++- .../ConnectionSupervisorActor.java | 17 +- .../ConnectivityMongoEventAdapter.java | 30 +- .../src/main/resources/connectivity.conf | 64 +++- ...ivityServiceGlobalCommandRegistryTest.java | 4 +- ...ctivityServiceGlobalEventRegistryTest.java | 4 +- .../service/messaging/TestConstants.java | 4 +- .../ConnectionPersistenceActorTest.java | 21 +- ...onnectionPersistenceOperationsActorIT.java | 5 +- .../_data/sidebars/ditto_sidebar.yml | 7 + .../src/main/resources/_data/tags.yml | 1 + ...streaming-subscription-failed-payload.json | 35 ++ ...l-streaming-subscription-next-payload.json | 16 + ...treaming-subscription-request-payload.json | 17 + ...ubscribe-for-persisted-events-payload.json | 26 ++ .../protocol-streaming-subscriptionid.json | 13 + .../resources/pages/ditto/basic-history.md | 178 +++++++++ .../pages/ditto/installation-operating.md | 85 ++++- ...ol-specification-streaming-subscription.md | 168 +++++++++ .../pages/ditto/protocol-specification.md | 35 +- .../main/resources/pages/tags/tag_history.md | 9 + .../EdgeCommandForwarderActor.java | 37 +- .../streaming/StreamingSubscriptionActor.java | 285 +++++++++++++++ .../StreamingSubscriptionManager.java | 205 +++++++++++ .../routes/sse/ThingsSseRouteBuilder.java | 131 +++++-- .../actors/SessionedJsonifiable.java | 16 +- .../streaming/actors/StreamingActor.java | 11 +- .../actors/StreamingSessionActor.java | 63 ++++ .../service/src/main/resources/gateway.conf | 4 +- ...tewayServiceGlobalCommandRegistryTest.java | 4 +- ...GatewayServiceGlobalEventRegistryTest.java | 4 +- ...mingSessionActorHeaderInteractionTest.java | 5 +- .../actors/StreamingSessionActorTest.java | 1 + .../src/main/resources/ditto-devops.conf | 4 + .../mongo/AbstractMongoEventAdapter.java | 48 ++- .../mongo/config/DefaultEventConfig.java | 84 +++++ .../mongo/config/DefaultSnapshotConfig.java | 2 +- .../persistence/mongo/config/EventConfig.java | 71 ++++ .../mongo/streaming/MongoReadJournal.java | 155 ++++++-- .../mongo/streaming/SnapshotFilter.java | 58 ++- .../mongo/config/DefaultEventConfigTest.java | 75 ++++ .../src/test/resources/event-test.conf | 6 + .../resources/mongo-read-journal-test.conf | 11 + .../persistence/src/test/resources/test.conf | 11 + .../AbstractPersistenceActor.java | 271 +++++++++++++- .../AbstractPersistenceSupervisor.java | 99 ++++- .../persistentactors/cleanup/Cleanup.java | 17 +- .../cleanup/CleanupConfig.java | 18 +- .../cleanup/DefaultCleanupConfig.java | 37 +- .../events/AbstractEventStrategies.java | 4 +- .../events/EventStrategy.java | 4 +- .../persistentactors/cleanup/CleanupTest.java | 13 +- .../persistentactors/cleanup/CreditsTest.java | 5 +- .../ditto/policies/model/WithPolicyId.java | 28 ++ .../model/signals/commands/PolicyCommand.java | 12 +- .../commands/PolicyCommandResponse.java | 9 +- .../PolicyHistoryNotAccessibleException.java | 177 +++++++++ .../signals/events/AbstractPolicyEvent.java | 7 +- .../model/signals/events/PolicyEvent.java | 10 +- .../events/SubjectsDeletedPartially.java | 10 +- .../events/SubjectsModifiedPartially.java | 10 +- .../common/config/DefaultPolicyConfig.java | 17 +- .../service/common/config/PolicyConfig.java | 8 + .../enforcement/PolicyCommandEnforcement.java | 83 +++-- .../actors/PolicyEnforcerActor.java | 2 +- .../actors/PolicyPersistenceActor.java | 28 +- .../actors/PolicySupervisorActor.java | 17 +- .../events/PolicyEventStrategies.java | 2 +- .../AbstractPolicyMongoEventAdapter.java | 19 +- .../DefaultPolicyMongoEventAdapter.java | 4 +- .../service/starter/PoliciesRootActor.java | 10 +- .../service/src/main/resources/policies.conf | 57 +++ .../PolicyCommandEnforcementTest.java | 3 +- ...olicyPersistenceActorSnapshottingTest.java | 4 +- .../actors/PolicyPersistenceActorTest.java | 10 +- .../PolicyPersistenceOperationsActorIT.java | 6 +- .../actors/PolicySupervisorActorTest.java | 11 +- ...iciesServiceGlobalCommandRegistryTest.java | 4 +- ...oliciesServiceGlobalEventRegistryTest.java | 4 +- .../ditto/protocol/ImmutablePayload.java | 8 +- .../ditto/protocol/ImmutableTopicPath.java | 87 ++++- .../ditto/protocol/ProtocolFactory.java | 34 ++ .../protocol/StreamingTopicPathBuilder.java | 78 ++++ .../org/eclipse/ditto/protocol/TopicPath.java | 100 ++++- .../ditto/protocol/TopicPathBuilder.java | 9 + .../AbstractStreamingMessageAdapter.java | 69 ++++ .../ditto/protocol/adapter/Adapter.java | 11 + .../adapter/AdapterResolverBySignal.java | 24 +- .../adapter/DefaultAdapterResolver.java | 10 +- .../adapter/DittoProtocolAdapter.java | 20 +- .../protocol/adapter/ProtocolAdapter.java | 10 +- .../StreamingSubscriptionCommandAdapter.java | 78 ++++ .../StreamingSubscriptionEventAdapter.java | 88 +++++ .../adapter/connectivity/package-info.java | 14 + .../DefaultPolicyCommandAdapterProvider.java | 11 +- .../adapter/policies/PolicyEventAdapter.java | 63 ++++ .../PolicyCommandAdapterProvider.java | 4 +- .../mapper/AbstractCommandSignalMapper.java | 4 +- .../mapper/PolicyEventSignalMapper.java | 92 +++++ .../protocol/mapper/SignalMapperFactory.java | 15 + ...eamingSubscriptionCommandSignalMapper.java | 137 +++++++ ...treamingSubscriptionEventSignalMapper.java | 108 ++++++ .../AbstractPolicyMappingStrategies.java | 34 ++ ...treamingSubscriptionMappingStrategies.java | 72 ++++ .../MappingStrategiesFactory.java | 13 + .../PolicyEventMappingStrategies.java | 239 ++++++++++++ ...gSubscriptionCommandMappingStrategies.java | 103 ++++++ ...ingSubscriptionEventMappingStrategies.java | 79 ++++ ...DittoProtocolAdapterParameterizedTest.java | 9 +- .../InvalidThingFieldSelectionException.java | 6 +- .../things/model/ThingFieldSelector.java | 2 +- .../ditto/things/model/WithThingId.java | 2 +- .../ThingHistoryNotAccessibleException.java | 177 +++++++++ .../common/config/DefaultThingConfig.java | 14 +- .../service/common/config/ThingConfig.java | 8 + .../StreamRequestingCommandEnforcement.java | 89 +++++ .../service/enforcement/ThingEnforcement.java | 3 +- .../actors/ThingPersistenceActor.java | 22 +- .../ThingPersistenceActorPropsFactory.java | 8 +- .../actors/ThingSupervisorActor.java | 24 +- ...hingsPersistenceStreamingActorCreator.java | 10 +- .../events/ThingEventStrategies.java | 4 +- .../serializer/ThingMongoEventAdapter.java | 34 +- ...aultThingPersistenceActorPropsFactory.java | 6 +- .../service/starter/ThingsRootActor.java | 12 +- things/service/src/main/resources/things.conf | 60 +++ .../AbstractThingEnforcementTest.java | 4 +- .../actors/PersistenceActorTestBase.java | 15 +- ...ThingPersistenceActorSnapshottingTest.java | 12 +- .../actors/ThingPersistenceActorTest.java | 5 +- .../ThingPersistenceOperationsActorIT.java | 12 +- .../ThingMongoEventAdapterTest.java | 15 +- ...hingsServiceGlobalCommandRegistryTest.java | 3 + .../ThingsServiceGlobalEventRegistryTest.java | 7 +- ...earchServiceGlobalCommandRegistryTest.java | 4 +- ...sSearchServiceGlobalEventRegistryTest.java | 4 +- 179 files changed, 7870 insertions(+), 401 deletions(-) create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/WithStreamingSubscriptionId.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionNotFoundException.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionProtocolErrorException.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionTimeoutException.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/AbstractStreamingSubscriptionCommand.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscription.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscription.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/StreamingSubscriptionCommand.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEvents.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/package-info.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/AbstractStreamingSubscriptionEvent.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionComplete.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreated.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionEvent.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailed.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNext.java create mode 100755 base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/package-info.java create mode 100755 base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscriptionTest.java create mode 100755 base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscriptionTest.java create mode 100755 base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEventsTest.java create mode 100644 base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCompleteTest.java create mode 100644 base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreatedTest.java create mode 100644 base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailedTest.java create mode 100644 base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNextTest.java create mode 100755 connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionHistoryNotAccessibleException.java create mode 100644 documentation/src/main/resources/jsonschema/protocol-streaming-subscription-failed-payload.json create mode 100644 documentation/src/main/resources/jsonschema/protocol-streaming-subscription-next-payload.json create mode 100644 documentation/src/main/resources/jsonschema/protocol-streaming-subscription-request-payload.json create mode 100644 documentation/src/main/resources/jsonschema/protocol-streaming-subscription-subscribe-for-persisted-events-payload.json create mode 100644 documentation/src/main/resources/jsonschema/protocol-streaming-subscriptionid.json create mode 100644 documentation/src/main/resources/pages/ditto/basic-history.md create mode 100644 documentation/src/main/resources/pages/ditto/protocol-specification-streaming-subscription.md create mode 100644 documentation/src/main/resources/pages/tags/tag_history.md create mode 100644 edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionActor.java create mode 100644 edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionManager.java create mode 100644 internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfig.java create mode 100644 internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/EventConfig.java create mode 100644 internal/utils/persistence/src/test/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfigTest.java create mode 100644 internal/utils/persistence/src/test/resources/event-test.conf create mode 100644 policies/model/src/main/java/org/eclipse/ditto/policies/model/WithPolicyId.java create mode 100755 policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyHistoryNotAccessibleException.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/StreamingTopicPathBuilder.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AbstractStreamingMessageAdapter.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionCommandAdapter.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionEventAdapter.java create mode 100755 protocol/src/main/java/org/eclipse/ditto/protocol/adapter/connectivity/package-info.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/PolicyEventAdapter.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/mapper/PolicyEventSignalMapper.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionCommandSignalMapper.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionEventSignalMapper.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractStreamingSubscriptionMappingStrategies.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/PolicyEventMappingStrategies.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionCommandMappingStrategies.java create mode 100644 protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionEventMappingStrategies.java create mode 100755 things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingHistoryNotAccessibleException.java create mode 100644 things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/StreamRequestingCommandEnforcement.java diff --git a/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceResponseTest.java b/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceResponseTest.java index b46a3584672..3e5406a7965 100644 --- a/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceResponseTest.java +++ b/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceResponseTest.java @@ -20,6 +20,7 @@ import org.eclipse.ditto.base.model.common.HttpStatus; import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.entity.type.EntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; @@ -33,7 +34,7 @@ */ public final class CleanupPersistenceResponseTest { - private static final EntityId ID = EntityId.of(EntityType.of("thing"), "eclipse:ditto"); + private static final EntityId ID = NamespacedEntityId.of(EntityType.of("thing"), "eclipse:ditto"); private static final JsonObject KNOWN_JSON = JsonObject.newBuilder() .set(CommandResponse.JsonFields.TYPE, CleanupPersistenceResponse.TYPE) .set(CleanupCommandResponse.JsonFields.ENTITY_TYPE, ID.getEntityType().toString()) diff --git a/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceTest.java b/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceTest.java index c1de951f4bd..43ba4297412 100644 --- a/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceTest.java +++ b/base/api/src/test/java/org/eclipse/ditto/base/api/persistence/cleanup/CleanupPersistenceTest.java @@ -18,6 +18,7 @@ import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.entity.type.EntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.signals.commands.Command; @@ -31,7 +32,7 @@ */ public class CleanupPersistenceTest { - private static final EntityId ID = EntityId.of(EntityType.of("thing"), "eclipse:ditto"); + private static final EntityId ID = NamespacedEntityId.of(EntityType.of("thing"), "eclipse:ditto"); private static final JsonObject KNOWN_JSON = JsonObject.newBuilder() .set(Command.JsonFields.TYPE, CleanupPersistence.TYPE) .set(CleanupCommand.JsonFields.ENTITY_TYPE, ID.getEntityType().toString()) diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/EntityId.java b/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/EntityId.java index 32566d5fd02..47f71909a7a 100644 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/EntityId.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/EntityId.java @@ -35,7 +35,13 @@ public interface EntityId extends CharSequence, Comparable { */ static EntityId of(final EntityType entityType, final CharSequence entityId) { final EntityIds entityIds = EntityIds.getInstance(); - return entityIds.getEntityId(entityType, entityId); + try { + // most entity ids are namespaces, so try that first + return entityIds.getNamespacedEntityId(entityType, entityId); + } catch (final NamespacedEntityIdInvalidException namespacedEntityIdInvalidException) { + // only in the exceptional case, fall back to non-namespaced flavor: + return entityIds.getEntityId(entityType, entityId); + } } @Override diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java index b8f520c80a4..c01d45e9b01 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java @@ -349,7 +349,7 @@ public enum DittoHeaderDefinition implements HeaderDefinition { * * @since 3.0.0 */ - DITTO_METADATA("ditto-metadata", JsonObject.class, false, true, HeaderValueValidators.getNoOpValidator()), + DITTO_METADATA("ditto-metadata", JsonObject.class, false, true, HeaderValueValidators.getJsonObjectValidator()), /** * Header definition for allowing the policy lockout (i.e. a subject can create a policy without having WRITE @@ -484,7 +484,43 @@ public enum DittoHeaderDefinition implements HeaderDefinition { Boolean.class, false, true, - HeaderValueValidators.getBooleanValidator()); + HeaderValueValidators.getBooleanValidator()), + + /** + * Header containing a specific historical revision to retrieve when retrieving a persisted entity + * (thing/policy/connection). + * + * @since 3.2.0 + */ + AT_HISTORICAL_REVISION("at-historical-revision", + Long.class, + true, + false, + HeaderValueValidators.getLongValidator()), + + /** + * Header containing a specific historical timestamp to retrieve when retrieving a persisted entity + * (thing/policy/connection). + * + * @since 3.2.0 + */ + AT_HISTORICAL_TIMESTAMP("at-historical-timestamp", + String.class, + true, + false, + HeaderValueValidators.getNoOpValidator()), + + /** + * Header containing retrieved historical headers to be returned for e.g. a historical retrieve command. + * Useful for audit-log information, e.g. which "originator" did a change to a thing/policy/connection. + * + * @since 3.2.0 + */ + HISTORICAL_HEADERS("historical-headers", + JsonObject.class, + false, + true, + HeaderValueValidators.getJsonObjectValidator()); /** * Map to speed up lookup of header definition by key. diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java index 07ee3bcc1d8..7bfc5bb5c9c 100644 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java @@ -33,6 +33,12 @@ public final class FeatureToggle { */ public static final String WOT_INTEGRATION_ENABLED = "ditto.devops.feature.wot-integration-enabled"; + /** + * System property name of the property defining whether the historical API access is enabled. + * @since 3.2.0 + */ + public static final String HISTORICAL_APIS_ENABLED = "ditto.devops.feature.historical-apis-enabled"; + /** * Resolves the system property {@value MERGE_THINGS_ENABLED}. */ @@ -43,6 +49,11 @@ public final class FeatureToggle { */ private static final boolean IS_WOT_INTEGRATION_ENABLED = resolveProperty(WOT_INTEGRATION_ENABLED); + /** + * Resolves the system property {@value HISTORICAL_APIS_ENABLED}. + */ + private static final boolean IS_HISTORICAL_APIS_ENABLED = resolveProperty(HISTORICAL_APIS_ENABLED); + private static boolean resolveProperty(final String propertyName) { final String propertyValue = System.getProperty(propertyName, Boolean.TRUE.toString()); return !Boolean.FALSE.toString().equalsIgnoreCase(propertyValue); @@ -101,4 +112,35 @@ public static DittoHeaders checkWotIntegrationFeatureEnabled(final String signal public static boolean isWotIntegrationFeatureEnabled() { return IS_WOT_INTEGRATION_ENABLED; } + + /** + * Checks if the historical API access feature is enabled based on the system property {@value HISTORICAL_APIS_ENABLED}. + * + * @param signal the name of the signal that was supposed to be processed + * @param dittoHeaders headers used to build exception + * @return the unmodified headers parameters + * @throws UnsupportedSignalException if the system property + * {@value HISTORICAL_APIS_ENABLED} resolves to {@code false} + * @since 3.2.0 + */ + public static DittoHeaders checkHistoricalApiAccessFeatureEnabled(final String signal, final DittoHeaders dittoHeaders) { + if (!isHistoricalApiAccessFeatureEnabled()) { + throw UnsupportedSignalException + .newBuilder(signal) + .dittoHeaders(dittoHeaders) + .build(); + } + return dittoHeaders; + } + + /** + * Returns whether the historical API access feature is enabled based on the system property + * {@value HISTORICAL_APIS_ENABLED}. + * + * @return whether the historical API access feature is enabled or not. + * @since 3.2.0 + */ + public static boolean isHistoricalApiAccessFeatureEnabled() { + return IS_HISTORICAL_APIS_ENABLED; + } } diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/WithStreamingSubscriptionId.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/WithStreamingSubscriptionId.java new file mode 100755 index 00000000000..a6810d5113a --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/WithStreamingSubscriptionId.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals; + +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; + +/** + * Interface of streaming commands/events addressing a particular session identified by a subscription ID. + * + * @since 3.2.0 + */ +public interface WithStreamingSubscriptionId> extends + StreamingSubscriptionCommand { + + /** + * Returns the subscriptionId identifying the session of this streaming signal. + * + * @return the subscriptionId. + */ + String getSubscriptionId(); + + /** + * Json fields of this command. + */ + final class JsonFields { + + /** + * JSON field for the streaming subscription ID. + */ + public static final JsonFieldDefinition SUBSCRIPTION_ID = + JsonFactory.newStringFieldDefinition("subscriptionId"); + + JsonFields() { + throw new AssertionError(); + } + + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/Command.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/Command.java index 4afd8b0ff4b..c4277fa6960 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/Command.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/Command.java @@ -159,7 +159,13 @@ enum Category { * Category of commands that are neither of the above 3 (query, modify, delete) but perform an action on the * entity. */ - ACTION; + ACTION, + + /** + * Category of commands that stream e.g. historical events. + * @since 3.2.0 + */ + STREAM; /** * Determines whether the passed {@code category} effectively modifies the targeted entity. diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionNotFoundException.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionNotFoundException.java new file mode 100755 index 00000000000..f0a4c9e3e03 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionNotFoundException.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.exceptions; + +import java.net.URI; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.exceptions.GeneralException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Error response for streaming subscription commands addressing a nonexistent subscription. + * + * @since 3.2.0 + */ +@JsonParsableException(errorCode = StreamingSubscriptionNotFoundException.ERROR_CODE) +public class StreamingSubscriptionNotFoundException extends DittoRuntimeException implements GeneralException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "streaming.subscription.not.found"; + + private static final HttpStatus STATUS_CODE = HttpStatus.NOT_FOUND; + + private StreamingSubscriptionNotFoundException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + + super(ERROR_CODE, STATUS_CODE, dittoHeaders, message, description, cause, href); + } + + /** + * Create a {@code StreamingSubscriptionNotFoundException}. + * + * @param subscriptionId ID of the nonexistent subscription. + * @param dittoHeaders the Ditto headers. + * @return the exception. + */ + public static StreamingSubscriptionNotFoundException of(final String subscriptionId, final DittoHeaders dittoHeaders) { + return new Builder() + .message(String.format("No subscription with ID '%s' exists.", subscriptionId)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * Constructs a new {@code StreamingSubscriptionNotFoundException} object with the exception message extracted from the + * given JSON object. + * + * @param jsonObject the JSON to read the {@link DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new StreamingSubscriptionNotFoundException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionNotFoundException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link StreamingSubscriptionNotFoundException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() {} + + @Override + protected StreamingSubscriptionNotFoundException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new StreamingSubscriptionNotFoundException(dittoHeaders, message, description, cause, href); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionProtocolErrorException.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionProtocolErrorException.java new file mode 100755 index 00000000000..5875e8660db --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionProtocolErrorException.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.exceptions; + +import java.net.URI; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.exceptions.GeneralException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Error response for subscriptions with no interaction for a long time. + * + * @since 3.2.0 + */ +@JsonParsableException(errorCode = StreamingSubscriptionProtocolErrorException.ERROR_CODE) +public class StreamingSubscriptionProtocolErrorException extends DittoRuntimeException implements GeneralException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "streaming.subscription.protocol.error"; + + private static final HttpStatus STATUS_CODE = HttpStatus.BAD_REQUEST; + + private StreamingSubscriptionProtocolErrorException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + + super(ERROR_CODE, STATUS_CODE, dittoHeaders, message, description, cause, href); + } + + /** + * Create a {@code StreamingSubscriptionProtocolErrorException}. + * + * @param cause the actual protocol error. + * @param dittoHeaders the Ditto headers. + * @return the exception. + */ + public static StreamingSubscriptionProtocolErrorException of(final Throwable cause, final DittoHeaders dittoHeaders) { + return new Builder() + .message(cause.getMessage()) + .cause(cause) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * Create an empty builder for this exception. + * + * @return an empty builder. + */ + public static DittoRuntimeExceptionBuilder newBuilder() { + return new Builder(); + } + + /** + * Constructs a new {@code StreamingSubscriptionProtocolErrorException} object with the exception message extracted from the + * given JSON object. + * + * @param jsonObject the JSON to read the {@link DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new StreamingSubscriptionProtocolErrorException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionProtocolErrorException fromJson(final JsonObject jsonObject, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link StreamingSubscriptionProtocolErrorException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() {} + + @Override + protected StreamingSubscriptionProtocolErrorException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new StreamingSubscriptionProtocolErrorException(dittoHeaders, message, description, cause, href); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionTimeoutException.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionTimeoutException.java new file mode 100755 index 00000000000..a55feff9a31 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/exceptions/StreamingSubscriptionTimeoutException.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.exceptions; + +import java.net.URI; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.exceptions.GeneralException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Error response for subscriptions with no interaction for a long time. + * + * @since 3.2.0 + */ +@JsonParsableException(errorCode = StreamingSubscriptionTimeoutException.ERROR_CODE) +public class StreamingSubscriptionTimeoutException extends DittoRuntimeException implements GeneralException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "streaming.subscription.timeout"; + + private static final HttpStatus STATUS_CODE = HttpStatus.REQUEST_TIMEOUT; + + private StreamingSubscriptionTimeoutException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + + super(ERROR_CODE, STATUS_CODE, dittoHeaders, message, description, cause, href); + } + + /** + * Create a {@code StreamingSubscriptionTimeoutException}. + * + * @param subscriptionId ID of the nonexistent subscription. + * @param dittoHeaders the Ditto headers. + * @return the exception. + */ + public static StreamingSubscriptionTimeoutException of(final String subscriptionId, final DittoHeaders dittoHeaders) { + return new Builder() + .message(String.format("The subscription '%s' stopped due to a lack of interaction.", subscriptionId)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * Constructs a new {@code StreamingSubscriptionTimeoutException} object with the exception message extracted from the + * given JSON object. + * + * @param jsonObject the JSON to read the {@link DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new StreamingSubscriptionTimeoutException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionTimeoutException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link StreamingSubscriptionTimeoutException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() {} + + @Override + protected StreamingSubscriptionTimeoutException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new StreamingSubscriptionTimeoutException(dittoHeaders, message, description, cause, href); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/AbstractStreamingSubscriptionCommand.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/AbstractStreamingSubscriptionCommand.java new file mode 100755 index 00000000000..1b8ff670875 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/AbstractStreamingSubscriptionCommand.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.util.Objects; +import java.util.function.Predicate; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.EntityIdJsonDeserializer; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.entity.type.EntityTypeJsonDeserializer; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.commands.AbstractCommand; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * Abstract base class for streaming commands. + * + * @param the type of the AbstractStreamingSubscriptionCommand + * @since 3.2.0 + */ +@Immutable +abstract class AbstractStreamingSubscriptionCommand> + extends AbstractCommand + implements StreamingSubscriptionCommand { + + protected final EntityId entityId; + protected final JsonPointer resourcePath; + + protected AbstractStreamingSubscriptionCommand(final String type, + final EntityId entityId, + final JsonPointer resourcePath, + final DittoHeaders dittoHeaders) { + + super(type, dittoHeaders); + this.entityId = checkNotNull(entityId, "entityId"); + this.resourcePath = checkNotNull(resourcePath, "resourcePath"); + } + + protected static EntityId deserializeEntityId(final JsonObject jsonObject) { + return EntityIdJsonDeserializer.deserializeEntityId(jsonObject, + StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_ID, + EntityTypeJsonDeserializer.deserializeEntityType(jsonObject, + StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_TYPE)); + } + + @Override + public Category getCategory() { + return Category.STREAM; + } + + @Override + public EntityId getEntityId() { + return entityId; + } + + @Override + public EntityType getEntityType() { + return entityId.getEntityType(); + } + + @Override + public JsonPointer getResourcePath() { + return resourcePath; + } + + @Override + public String getResourceType() { + return getEntityType().toString(); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, + final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + final Predicate predicate = schemaVersion.and(thePredicate); + jsonObjectBuilder.set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_TYPE, + entityId.getEntityType().toString(), predicate); + jsonObjectBuilder.set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_ID, + entityId.toString(), predicate); + jsonObjectBuilder.set(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH, + resourcePath.toString(), predicate); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), entityId, resourcePath); + } + + @SuppressWarnings({"squid:MethodCyclomaticComplexity", "squid:S1067", "OverlyComplexMethod"}) + @Override + public boolean equals(@Nullable final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final AbstractStreamingSubscriptionCommand other = (AbstractStreamingSubscriptionCommand) obj; + + return other.canEqual(this) && + super.equals(other) && + Objects.equals(entityId, other.entityId) && + Objects.equals(resourcePath, other.resourcePath); + } + + @Override + protected boolean canEqual(@Nullable final Object other) { + return other instanceof AbstractStreamingSubscriptionCommand; + } + + @Override + public String toString() { + return super.toString() + + ", entityId=" + entityId + + ", entityType=" + getEntityType() + + ", resourcePath=" + resourcePath; + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscription.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscription.java new file mode 100755 index 00000000000..8586c5cd5fe --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscription.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import java.util.Objects; +import java.util.function.Predicate; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableCommand; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.commands.CommandJsonDeserializer; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * Command for cancelling a subscription of streaming results. + * Corresponds to the reactive-streams signal {@code Subscription#cancel()}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableCommand(typePrefix = StreamingSubscriptionCommand.TYPE_PREFIX, name = CancelStreamingSubscription.NAME) +public final class CancelStreamingSubscription extends AbstractStreamingSubscriptionCommand + implements WithStreamingSubscriptionId { + + /** + * Name of the command. + */ + public static final String NAME = "cancel"; + + /** + * Type of this command. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final String subscriptionId; + + private CancelStreamingSubscription(final EntityId entityId, + final JsonPointer resourcePath, + final String subscriptionId, + final DittoHeaders dittoHeaders) { + super(TYPE, entityId, resourcePath, dittoHeaders); + this.subscriptionId = subscriptionId; + } + + /** + * Returns a new instance of the command. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream. + * @param subscriptionId ID of the subscription to cancel. + * @param dittoHeaders the headers of the command. + * @return a new command to cancel a subscription. + * @throws NullPointerException if {@code dittoHeaders} is {@code null}. + */ + public static CancelStreamingSubscription of(final EntityId entityId, + final JsonPointer resourcePath, + final String subscriptionId, + final DittoHeaders dittoHeaders) { + return new CancelStreamingSubscription(entityId, resourcePath, subscriptionId, dittoHeaders); + } + + /** + * Creates a new {@code CancelStreamingSubscription} from a JSON object. + * + * @param jsonObject the JSON object of which the command is to be created. + * @param dittoHeaders the headers of the command. + * @return the command. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static CancelStreamingSubscription fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new CommandJsonDeserializer(TYPE, jsonObject).deserialize(() -> + new CancelStreamingSubscription(deserializeEntityId(jsonObject), + JsonPointer.of( + jsonObject.getValueOrThrow(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH)), + jsonObject.getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID), + dittoHeaders + ) + ); + } + + @Override + public String getSubscriptionId() { + return subscriptionId; + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + super.appendPayload(jsonObjectBuilder, schemaVersion, thePredicate); + jsonObjectBuilder.set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, subscriptionId); + } + + @Override + public CancelStreamingSubscription setDittoHeaders(final DittoHeaders dittoHeaders) { + return new CancelStreamingSubscription(entityId, resourcePath, subscriptionId, dittoHeaders); + } + + @Override + public boolean equals(@Nullable final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CancelStreamingSubscription)) { + return false; + } + if (!super.equals(o)) { + return false; + } + final CancelStreamingSubscription that = (CancelStreamingSubscription) o; + return Objects.equals(subscriptionId, that.subscriptionId); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), subscriptionId); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[" + + super.toString() + + ", subscriptionId=" + subscriptionId + + ']'; + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscription.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscription.java new file mode 100755 index 00000000000..4a72cde4204 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscription.java @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import java.util.Objects; +import java.util.function.Predicate; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableCommand; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.commands.CommandJsonDeserializer; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * Command for requesting items from a subscription of streaming results. + * Corresponds to the reactive-streams signal {@code Subscription#request(long)}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableCommand(typePrefix = StreamingSubscriptionCommand.TYPE_PREFIX, name = RequestFromStreamingSubscription.NAME) +public final class RequestFromStreamingSubscription extends AbstractStreamingSubscriptionCommand + implements WithStreamingSubscriptionId { + + /** + * Name of the command. + */ + public static final String NAME = "request"; + + /** + * Type of this command. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final String subscriptionId; + private final long demand; + + private RequestFromStreamingSubscription(final EntityId entityId, + final JsonPointer resourcePath, + final String subscriptionId, + final long demand, + final DittoHeaders dittoHeaders) { + + super(TYPE, entityId, resourcePath, dittoHeaders); + this.subscriptionId = subscriptionId; + this.demand = demand; + } + + /** + * Returns a new instance of the command. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream. + * @param subscriptionId ID of the subscription to request from. + * @param demand how many pages to request. + * @param dittoHeaders the headers of the command. + * @return a new command to request from a subscription. + * @throws NullPointerException if {@code dittoHeaders} is {@code null}. + */ + public static RequestFromStreamingSubscription of(final EntityId entityId, + final JsonPointer resourcePath, + final String subscriptionId, + final long demand, + final DittoHeaders dittoHeaders) { + return new RequestFromStreamingSubscription(entityId, resourcePath, subscriptionId, demand, dittoHeaders); + } + + /** + * Creates a new {@code RequestSubscription} from a JSON object. + * + * @param jsonObject the JSON object of which the command is to be created. + * @param dittoHeaders the headers of the command. + * @return the command. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static RequestFromStreamingSubscription fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new CommandJsonDeserializer(TYPE, jsonObject).deserialize(() -> + new RequestFromStreamingSubscription(deserializeEntityId(jsonObject), + JsonPointer.of( + jsonObject.getValueOrThrow(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH)), + jsonObject.getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID), + jsonObject.getValueOrThrow(JsonFields.DEMAND), + dittoHeaders + ) + ); + } + + + @Override + public String getSubscriptionId() { + return subscriptionId; + } + + + /** + * Returns the demand which is to be included in the JSON of the retrieved entity. + * + * @return the demand. + */ + public long getDemand() { + return demand; + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + super.appendPayload(jsonObjectBuilder, schemaVersion, thePredicate); + jsonObjectBuilder.set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, subscriptionId); + jsonObjectBuilder.set(JsonFields.DEMAND, demand); + } + + @Override + public String getResourceType() { + return getEntityType().toString(); + } + + @Override + public RequestFromStreamingSubscription setDittoHeaders(final DittoHeaders dittoHeaders) { + return new RequestFromStreamingSubscription(entityId, resourcePath, subscriptionId, demand, dittoHeaders); + } + + @Override + public boolean equals(@Nullable final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RequestFromStreamingSubscription)) { + return false; + } + if (!super.equals(o)) { + return false; + } + final RequestFromStreamingSubscription that = (RequestFromStreamingSubscription) o; + return Objects.equals(subscriptionId, that.subscriptionId) && demand == that.demand; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), subscriptionId, demand); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[" + + super.toString() + + ", subscriptionId=" + subscriptionId + + ", demand=" + demand + + ']'; + } + + /** + * JSON fields of this command. + */ + public static final class JsonFields { + + /** + * JSON field for number of pages demanded by this command. + */ + public static final JsonFieldDefinition DEMAND = JsonFactory.newLongFieldDefinition("demand"); + + JsonFields() { + throw new AssertionError(); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/StreamingSubscriptionCommand.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/StreamingSubscriptionCommand.java new file mode 100755 index 00000000000..f57d240d934 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/StreamingSubscriptionCommand.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.eclipse.ditto.base.model.json.FieldType.REGULAR; +import static org.eclipse.ditto.base.model.json.JsonSchemaVersion.V_2; + +import org.eclipse.ditto.base.model.entity.type.WithEntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.SignalWithEntityId; +import org.eclipse.ditto.base.model.signals.WithResource; +import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; + +/** + * Aggregates all {@link Command}s which request a stream (e.g. a {@code SourceRef}) of + * {@link org.eclipse.ditto.base.model.signals.Signal}s to subscribe for. + * + * @param the type of the implementing class. + * @since 3.2.0 + */ +public interface StreamingSubscriptionCommand> extends Command, + WithEntityType, SignalWithEntityId, WithResource { + + /** + * Resource type of streaming subscription commands. + */ + String RESOURCE_TYPE = "streaming.subscription"; + + /** + * Type Prefix of Streaming commands. + */ + String TYPE_PREFIX = RESOURCE_TYPE + "." + TYPE_QUALIFIER + ":"; + + @Override + default String getTypePrefix() { + return TYPE_PREFIX; + } + + @Override + T setDittoHeaders(DittoHeaders dittoHeaders); + + /** + * This class contains definitions for all specific fields of this command's JSON representation. + */ + final class JsonFields { + + private JsonFields() { + throw new AssertionError(); + } + + public static final JsonFieldDefinition JSON_ENTITY_ID = + JsonFactory.newStringFieldDefinition("entityId", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_ENTITY_TYPE = + JsonFactory.newStringFieldDefinition("entityType", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_RESOURCE_PATH = + JsonFactory.newStringFieldDefinition("resourcePath", REGULAR, V_2); + + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEvents.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEvents.java new file mode 100755 index 00000000000..aeca3a7738f --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEvents.java @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.eclipse.ditto.base.model.json.FieldType.REGULAR; +import static org.eclipse.ditto.base.model.json.JsonSchemaVersion.V_2; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableCommand; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * Command which starts a stream of journal entries as persisted events for a given EntityId. + * Corresponds to the reactive-streams signal {@code Publisher#subscribe(Subscriber)}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableCommand(typePrefix = StreamingSubscriptionCommand.TYPE_PREFIX, name = SubscribeForPersistedEvents.NAME) +public final class SubscribeForPersistedEvents extends AbstractStreamingSubscriptionCommand + implements StreamingSubscriptionCommand { + + /** + * The name of this streaming subscription command. + */ + public static final String NAME = "subscribeForPersistedEvents"; + + /** + * Type of this command. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final long fromHistoricalRevision; + private final long toHistoricalRevision; + + @Nullable private final Instant fromHistoricalTimestamp; + @Nullable private final Instant toHistoricalTimestamp; + @Nullable private final String prefix; + + private SubscribeForPersistedEvents(final EntityId entityId, + final JsonPointer resourcePath, + final long fromHistoricalRevision, + final long toHistoricalRevision, + @Nullable final Instant fromHistoricalTimestamp, + @Nullable final Instant toHistoricalTimestamp, + @Nullable final String prefix, + final DittoHeaders dittoHeaders) { + + super(TYPE, entityId, resourcePath, dittoHeaders); + this.fromHistoricalRevision = fromHistoricalRevision; + this.toHistoricalRevision = toHistoricalRevision; + this.fromHistoricalTimestamp = fromHistoricalTimestamp; + this.toHistoricalTimestamp = toHistoricalTimestamp; + this.prefix = prefix; + } + + /** + * Creates a new {@code SudoStreamSnapshots} command based on "from" and "to" {@code long} revisions. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream events. + * @param fromHistoricalRevision the revision to start the streaming from. + * @param toHistoricalRevision the revision to stop the streaming at. + * @param dittoHeaders the command headers of the request. + * @return the command. + * @throws NullPointerException if any non-nullable argument is {@code null}. + */ + public static SubscribeForPersistedEvents of(final EntityId entityId, + final JsonPointer resourcePath, + final long fromHistoricalRevision, + final long toHistoricalRevision, + final DittoHeaders dittoHeaders) { + + return new SubscribeForPersistedEvents(entityId, + resourcePath, + fromHistoricalRevision, + toHistoricalRevision, + null, + null, + null, + dittoHeaders); + } + + /** + * Creates a new {@code SudoStreamSnapshots} command based on "from" and "to" {@code Instant} timestamps. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream events. + * @param fromHistoricalTimestamp the timestamp to start the streaming from. + * @param toHistoricalTimestamp the timestamp to stop the streaming at. + * @param dittoHeaders the command headers of the request. + * @return the command. + * @throws NullPointerException if any non-nullable argument is {@code null}. + */ + public static SubscribeForPersistedEvents of(final EntityId entityId, + final JsonPointer resourcePath, + @Nullable final Instant fromHistoricalTimestamp, + @Nullable final Instant toHistoricalTimestamp, + final DittoHeaders dittoHeaders) { + + return new SubscribeForPersistedEvents(entityId, + resourcePath, + 0L, + Long.MAX_VALUE, + fromHistoricalTimestamp, + toHistoricalTimestamp, + null, + dittoHeaders); + } + + /** + * Creates a new {@code SudoStreamSnapshots} command based on "from" and "to" {@code Instant} timestamps. + * + * @param entityId the entityId that should be streamed. + * @param resourcePath the resource path for which to stream events. + * @param fromHistoricalRevision the revision to start the streaming from. + * @param toHistoricalRevision the revision to stop the streaming at. + * @param fromHistoricalTimestamp the timestamp to start the streaming from. + * @param toHistoricalTimestamp the timestamp to stop the streaming at. + * @param dittoHeaders the command headers of the request. + * @return the command. + * @throws NullPointerException if any non-nullable argument is {@code null}. + */ + public static SubscribeForPersistedEvents of(final EntityId entityId, + final JsonPointer resourcePath, + @Nullable final Long fromHistoricalRevision, + @Nullable final Long toHistoricalRevision, + @Nullable final Instant fromHistoricalTimestamp, + @Nullable final Instant toHistoricalTimestamp, + final DittoHeaders dittoHeaders) { + + return new SubscribeForPersistedEvents(entityId, + resourcePath, + null != fromHistoricalRevision ? fromHistoricalRevision : 0L, + null != toHistoricalRevision ? toHistoricalRevision : Long.MAX_VALUE, + fromHistoricalTimestamp, + toHistoricalTimestamp, + null, + dittoHeaders); + } + + /** + * Deserializes a {@code SubscribeForPersistedEvents} from the specified {@link JsonObject} argument. + * + * @param jsonObject the JSON object to be deserialized. + * @return the deserialized {@code SubscribeForPersistedEvents}. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if {@code jsonObject} did not contain all required + * fields. + * @throws org.eclipse.ditto.json.JsonParseException if {@code jsonObject} was not in the expected format. + */ + public static SubscribeForPersistedEvents fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new SubscribeForPersistedEvents(deserializeEntityId(jsonObject), + JsonPointer.of(jsonObject.getValueOrThrow(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH)), + jsonObject.getValueOrThrow(JsonFields.JSON_FROM_HISTORICAL_REVISION), + jsonObject.getValueOrThrow(JsonFields.JSON_TO_HISTORICAL_REVISION), + jsonObject.getValue(JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP).map(Instant::parse).orElse(null), + jsonObject.getValue(JsonFields.JSON_TO_HISTORICAL_TIMESTAMP).map(Instant::parse).orElse(null), + jsonObject.getValue(JsonFields.PREFIX).orElse(null), + dittoHeaders + ); + } + + /** + * Create a copy of this command with prefix set. The prefix is used to identify a streaming subscription manager + * if multiple are deployed in the cluster. + * + * @param prefix the subscription ID prefix. + * @return the new command. + */ + public SubscribeForPersistedEvents setPrefix(@Nullable final String prefix) { + return new SubscribeForPersistedEvents(entityId, resourcePath, fromHistoricalRevision, toHistoricalRevision, + fromHistoricalTimestamp, toHistoricalTimestamp, prefix, getDittoHeaders()); + } + + /** + * Returns the revision to start the streaming from. + * + * @return the revision to start the streaming from. + */ + public long getFromHistoricalRevision() { + return fromHistoricalRevision; + } + + /** + * Returns the timestamp to stop the streaming at. + * + * @return the timestamp to stop the streaming at. + */ + public long getToHistoricalRevision() { + return toHistoricalRevision; + } + + /** + * Returns the optional timestamp to start the streaming from. + * + * @return the optional timestamp to start the streaming from. + */ + public Optional getFromHistoricalTimestamp() { + return Optional.ofNullable(fromHistoricalTimestamp); + } + + /** + * Returns the optional timestamp to stop the streaming at. + * + * @return the optional timestamp to stop the streaming at. + */ + public Optional getToHistoricalTimestamp() { + return Optional.ofNullable(toHistoricalTimestamp); + } + + /** + * Get the prefix of subscription IDs. The prefix is used to identify a streaming subscription manager if multiple + * are deployed in the cluster. + * + * @return the subscription ID prefix. + */ + public Optional getPrefix() { + return Optional.ofNullable(prefix); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, + final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + super.appendPayload(jsonObjectBuilder, schemaVersion, thePredicate); + + final Predicate predicate = schemaVersion.and(thePredicate); + jsonObjectBuilder.set(JsonFields.JSON_FROM_HISTORICAL_REVISION, fromHistoricalRevision, predicate); + jsonObjectBuilder.set(JsonFields.JSON_TO_HISTORICAL_REVISION, toHistoricalRevision, predicate); + jsonObjectBuilder.set(JsonFields.JSON_TO_HISTORICAL_REVISION, toHistoricalRevision, predicate); + if (null != fromHistoricalTimestamp) { + jsonObjectBuilder.set(JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP, fromHistoricalTimestamp.toString(), + predicate); + } + if (null != toHistoricalTimestamp) { + jsonObjectBuilder.set(JsonFields.JSON_TO_HISTORICAL_TIMESTAMP, toHistoricalTimestamp.toString(), predicate); + } + getPrefix().ifPresent(thePrefix -> jsonObjectBuilder.set(JsonFields.PREFIX, thePrefix)); + } + + @Override + public String getTypePrefix() { + return TYPE_PREFIX; + } + + @Override + public SubscribeForPersistedEvents setDittoHeaders(final DittoHeaders dittoHeaders) { + return new SubscribeForPersistedEvents(entityId, resourcePath, fromHistoricalRevision, toHistoricalRevision, + fromHistoricalTimestamp, toHistoricalTimestamp, prefix, dittoHeaders); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), entityId, resourcePath, fromHistoricalRevision, toHistoricalRevision, + fromHistoricalTimestamp, toHistoricalTimestamp, prefix); + } + + @Override + public boolean equals(@Nullable final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final SubscribeForPersistedEvents that = (SubscribeForPersistedEvents) obj; + + return that.canEqual(this) && super.equals(that) && + fromHistoricalRevision == that.fromHistoricalRevision && + toHistoricalRevision == that.toHistoricalRevision && + Objects.equals(fromHistoricalTimestamp, that.fromHistoricalTimestamp) && + Objects.equals(toHistoricalTimestamp, that.toHistoricalTimestamp) && + Objects.equals(prefix, that.prefix); + } + + @Override + protected boolean canEqual(@Nullable final Object other) { + return other instanceof SubscribeForPersistedEvents; + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + + ", fromHistoricalRevision=" + fromHistoricalRevision + + ", toHistoricalRevision=" + toHistoricalRevision + + ", fromHistoricalTimestamp=" + fromHistoricalTimestamp + + ", toHistoricalTimestamp=" + toHistoricalTimestamp + + ", prefix=" + prefix + + "]"; + } + + /** + * This class contains definitions for all specific fields of this command's JSON representation. + */ + public static final class JsonFields { + + private JsonFields() { + throw new AssertionError(); + } + + public static final JsonFieldDefinition JSON_FROM_HISTORICAL_REVISION = + JsonFactory.newLongFieldDefinition("fromHistoricalRevision", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_TO_HISTORICAL_REVISION = + JsonFactory.newLongFieldDefinition("toHistoricalRevision", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_FROM_HISTORICAL_TIMESTAMP = + JsonFactory.newStringFieldDefinition("fromHistoricalTimestamp", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_TO_HISTORICAL_TIMESTAMP = + JsonFactory.newStringFieldDefinition("toHistoricalTimestamp", REGULAR, V_2); + + static final JsonFieldDefinition PREFIX = + JsonFactory.newStringFieldDefinition("prefix", REGULAR, V_2); + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/package-info.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/package-info.java new file mode 100755 index 00000000000..565442a816b --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/package-info.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.eclipse.ditto.utils.jsr305.annotations.AllValuesAreNonnullByDefault +package org.eclipse.ditto.base.model.signals.commands.streaming; + diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEvent.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEvent.java index 3f19feec46a..815c280930a 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEvent.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEvent.java @@ -98,6 +98,7 @@ public String getManifest() { @Override public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate thePredicate) { final Predicate predicate = schemaVersion.and(thePredicate); + final JsonObjectBuilder jsonObjectBuilder = JsonFactory.newObjectBuilder() // TYPE is included unconditionally: .set(JsonFields.TYPE, type) diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEventsourcedEvent.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEventsourcedEvent.java index af20d1b37e7..0276fa4d5e0 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEventsourcedEvent.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/AbstractEventsourcedEvent.java @@ -96,6 +96,7 @@ public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate< // it shall not invoke super.toJson(...) because in that case "appendPayloadAndBuild" would be invoked twice // and the order of the fields to appear in the JSON would not be controllable! final Predicate predicate = schemaVersion.and(thePredicate); + final JsonObjectBuilder jsonObjectBuilder = JsonFactory.newObjectBuilder() // TYPE + entityId is included unconditionally: .set(Event.JsonFields.TYPE, getType()) diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/EventJsonDeserializer.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/EventJsonDeserializer.java index 782ea697881..e8704a716b9 100755 --- a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/EventJsonDeserializer.java +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/EventJsonDeserializer.java @@ -22,13 +22,13 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; +import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.exceptions.DittoJsonException; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonMissingFieldException; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonParseException; import org.eclipse.ditto.json.JsonValue; -import org.eclipse.ditto.base.model.entity.metadata.Metadata; -import org.eclipse.ditto.base.model.exceptions.DittoJsonException; /** * This class helps to deserialize JSON to a sub-class of {@link Event}. Hereby this class extracts the values which diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/AbstractStreamingSubscriptionEvent.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/AbstractStreamingSubscriptionEvent.java new file mode 100755 index 00000000000..ce0089bc51d --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/AbstractStreamingSubscriptionEvent.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.EntityIdJsonDeserializer; +import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.entity.type.EntityTypeJsonDeserializer; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; + +/** + * Abstract base class of subscription events. Package-private. Not to be extended in user code. + * + * @param the type of the implementing class. + * @since 3.2.0 + */ +@Immutable +abstract class AbstractStreamingSubscriptionEvent> implements + StreamingSubscriptionEvent { + + private final String type; + private final String subscriptionId; + + private final EntityId entityId; + private final DittoHeaders dittoHeaders; + + /** + * Constructs a new {@code AbstractStreamingSubscriptionEvent} object. + * + * @param type the type of this event. + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @throws NullPointerException if any argument but {@code timestamp} is {@code null}. + */ + protected AbstractStreamingSubscriptionEvent(final String type, + final String subscriptionId, + final EntityId entityId, + final DittoHeaders dittoHeaders) { + + this.type = checkNotNull(type, "type"); + this.subscriptionId = checkNotNull(subscriptionId, "subscriptionId"); + this.entityId = checkNotNull(entityId, "entityId"); + this.dittoHeaders = checkNotNull(dittoHeaders, "dittoHeaders"); + } + + @Override + public String getSubscriptionId() { + return subscriptionId; + } + + @Override + public String getType() { + return type; + } + + @Override + public EntityId getEntityId() { + return entityId; + } + + @Override + public EntityType getEntityType() { + return entityId.getEntityType(); + } + + @Override + public Optional getTimestamp() { + // subscription events have no timestamp. + return Optional.empty(); + } + + @Override + public Optional getMetadata() { + return Optional.empty(); + } + + @Override + public DittoHeaders getDittoHeaders() { + return dittoHeaders; + } + + @Nonnull + @Override + public String getManifest() { + return getType(); + } + + @Override + public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate thePredicate) { + final JsonObjectBuilder jsonObjectBuilder = JsonFactory.newObjectBuilder() + // TYPE is included unconditionally + .set(Event.JsonFields.TYPE, type) + .set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, subscriptionId) + .set(StreamingSubscriptionEvent.JsonFields.JSON_ENTITY_ID, entityId.toString()) + .set(StreamingSubscriptionEvent.JsonFields.JSON_ENTITY_TYPE, entityId.getEntityType().toString()); + + appendPayload(jsonObjectBuilder); + + return jsonObjectBuilder.build(); + } + + protected static EntityId deserializeEntityId(final JsonObject jsonObject) { + return EntityIdJsonDeserializer.deserializeEntityId(jsonObject, + StreamingSubscriptionEvent.JsonFields.JSON_ENTITY_ID, + EntityTypeJsonDeserializer.deserializeEntityType(jsonObject, + StreamingSubscriptionEvent.JsonFields.JSON_ENTITY_TYPE)); + } + + /** + * Appends the event specific custom payload to the passed {@code jsonObjectBuilder}. + * + * @param jsonObjectBuilder the JsonObjectBuilder to add the custom payload to. + */ + protected abstract void appendPayload(final JsonObjectBuilder jsonObjectBuilder); + + @Override + public boolean equals(@Nullable final Object o) { + if (o != null && getClass() == o.getClass()) { + final AbstractStreamingSubscriptionEvent that = (AbstractStreamingSubscriptionEvent) o; + return Objects.equals(type, that.type) && + Objects.equals(subscriptionId, that.subscriptionId) && + Objects.equals(entityId, that.entityId) && + Objects.equals(dittoHeaders, that.dittoHeaders); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(type, subscriptionId, entityId, dittoHeaders); + } + + @Override + public String toString() { + return "type=" + type + + ", subscriptionId=" + subscriptionId + + ", entityId=" + entityId + + ", dittoHeaders=" + dittoHeaders; + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionComplete.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionComplete.java new file mode 100755 index 00000000000..e6bd039cddd --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionComplete.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableEvent; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * This event is emitted after all items of a subscription are sent. + * Corresponds to the reactive-streams signal {@code Subscriber#onComplete()}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableEvent(name = StreamingSubscriptionComplete.NAME, typePrefix = StreamingSubscriptionEvent.TYPE_PREFIX) +public final class StreamingSubscriptionComplete + extends AbstractStreamingSubscriptionEvent { + + /** + * Name of the event. + */ + public static final String NAME = "complete"; + + /** + * Type of this event. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private StreamingSubscriptionComplete(final String subscriptionId, final EntityId entityId, + final DittoHeaders dittoHeaders) { + super(TYPE, subscriptionId, entityId, dittoHeaders); + } + + /** + * Constructs a new {@code StreamingSubscriptionComplete} object. + * + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the StreamingSubscriptionComplete created. + * @throws NullPointerException if either argument is null. + */ + public static StreamingSubscriptionComplete of(final String subscriptionId, final EntityId entityId, + final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionComplete(subscriptionId, entityId, dittoHeaders); + } + + /** + * Creates a new {@code StreamingSubscriptionComplete} from a JSON object. + * + * @param jsonObject the JSON object from which a new StreamingSubscriptionComplete instance is to be created. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the {@code StreamingSubscriptionComplete} which was created from the given JSON object. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionComplete fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new EventJsonDeserializer(TYPE, jsonObject) + .deserialize((revision, timestamp, metadata) -> { + final String subscriptionId = jsonObject + .getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID); + final EntityId entityId = deserializeEntityId(jsonObject); + return new StreamingSubscriptionComplete(subscriptionId, entityId, dittoHeaders); + }); + } + + @Override + public JsonPointer getResourcePath() { + return JsonPointer.empty(); + } + + @Override + public StreamingSubscriptionComplete setDittoHeaders(final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionComplete(getSubscriptionId(), getEntityId(), dittoHeaders); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder) { + // nothing to add + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + "]"; + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreated.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreated.java new file mode 100755 index 00000000000..6c6ffc518e2 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreated.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableEvent; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * This event is emitted after a stream is established for items to be streamed in the back-end. + * Corresponds to the reactive-streams signal {@code Subscriber#onSubscribe(Subscription)}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableEvent(name = StreamingSubscriptionCreated.NAME, typePrefix = StreamingSubscriptionEvent.TYPE_PREFIX) +public final class StreamingSubscriptionCreated + extends AbstractStreamingSubscriptionEvent { + + /** + * Name of the event. + */ + public static final String NAME = "created"; + + /** + * Type of this event. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private StreamingSubscriptionCreated(final String subscriptionId, + final EntityId entityId, + final DittoHeaders dittoHeaders) { + super(TYPE, subscriptionId, entityId, dittoHeaders); + } + + /** + * Constructs a new {@code StreamingSubscriptionCreated} event. + * + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the event. + * @throws NullPointerException if either argument is null. + */ + public static StreamingSubscriptionCreated of(final String subscriptionId, + final EntityId entityId, + final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionCreated(subscriptionId, entityId, dittoHeaders); + } + + /** + * Creates a new {@code StreamingSubscriptionCreated} from a JSON object. + * + * @param jsonObject the JSON object from which a new StreamingSubscriptionCreated instance is to be created. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the {@code StreamingSubscriptionCreated} which was created from the given JSON object. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionCreated fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new EventJsonDeserializer(TYPE, jsonObject) + .deserialize((revision, timestamp, metadata) -> { + final String subscriptionId = jsonObject + .getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID); + final EntityId entityId = deserializeEntityId(jsonObject); + return new StreamingSubscriptionCreated(subscriptionId, entityId, dittoHeaders); + }); + } + + @Override + public JsonPointer getResourcePath() { + return JsonPointer.empty(); + } + + @Override + public StreamingSubscriptionCreated setDittoHeaders(final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionCreated(getSubscriptionId(), getEntityId(), dittoHeaders); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder) { + // nothing to add + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + "]"; + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionEvent.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionEvent.java new file mode 100755 index 00000000000..a009d0fded2 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionEvent.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.eclipse.ditto.base.model.json.FieldType.REGULAR; +import static org.eclipse.ditto.base.model.json.JsonSchemaVersion.V_2; + +import org.eclipse.ditto.base.model.entity.id.WithEntityId; +import org.eclipse.ditto.base.model.entity.type.WithEntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; + +/** + * Interface for all outgoing messages related to a subscription for streaming something. + * + * @param the type of the implementing class. + * @since 3.2.0 + */ +public interface StreamingSubscriptionEvent> extends Event, + WithEntityId, WithEntityType { + + /** + * Resource type of streaming subscription events. + */ + String RESOURCE_TYPE = "streaming.subscription"; + + /** + * Type Prefix of Streaming events. + */ + String TYPE_PREFIX = RESOURCE_TYPE + "." + TYPE_QUALIFIER + ":"; + + /** + * Returns the subscriptionId identifying the session of this streaming signal. + * + * @return the subscriptionId. + */ + String getSubscriptionId(); + + @Override + T setDittoHeaders(DittoHeaders dittoHeaders); + + @Override + default String getResourceType() { + return RESOURCE_TYPE; + } + + /** + * This class contains definitions for all specific fields of this event's JSON representation. + */ + final class JsonFields { + + private JsonFields() { + throw new AssertionError(); + } + + public static final JsonFieldDefinition JSON_ENTITY_ID = + JsonFactory.newStringFieldDefinition("entityId", REGULAR, V_2); + + public static final JsonFieldDefinition JSON_ENTITY_TYPE = + JsonFactory.newStringFieldDefinition("entityType", REGULAR, V_2); + + } + +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailed.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailed.java new file mode 100755 index 00000000000..4e4f878e34d --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailed.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableEvent; +import org.eclipse.ditto.base.model.signals.GlobalErrorRegistry; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; + +/** + * This event is emitted after a stream failed. + * Corresponds to the reactive-streams signal {@code Subscriber#onError()}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableEvent(name = StreamingSubscriptionFailed.NAME, typePrefix = StreamingSubscriptionEvent.TYPE_PREFIX) +public final class StreamingSubscriptionFailed extends AbstractStreamingSubscriptionEvent { + + /** + * Name of the event. + */ + public static final String NAME = "failed"; + + /** + * Type of this event. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final DittoRuntimeException error; + + private StreamingSubscriptionFailed(final String subscriptionId, + final EntityId entityId, + final DittoRuntimeException error, + final DittoHeaders dittoHeaders) { + super(TYPE, subscriptionId, entityId, dittoHeaders); + this.error = error; + } + + /** + * Constructs a new {@code StreamingSubscriptionFailed} object. + * + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param error the cause of the failure. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the StreamingSubscriptionFailed created. + * @throws NullPointerException if either argument is null. + */ + public static StreamingSubscriptionFailed of(final String subscriptionId, + final EntityId entityId, + final DittoRuntimeException error, + final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionFailed(subscriptionId, entityId, error, dittoHeaders); + } + + /** + * Creates a new {@code StreamingSubscriptionFailed} from a JSON object. + * + * @param jsonObject the JSON object from which a new StreamingSubscriptionFailed instance is to be created. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the {@code StreamingSubscriptionFailed} which was created from the given JSON object. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionFailed fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new EventJsonDeserializer(TYPE, jsonObject) + .deserialize((revision, timestamp, metadata) -> { + final String subscriptionId = + jsonObject.getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID); + final JsonObject errorJson = jsonObject.getValueOrThrow(JsonFields.ERROR); + final EntityId entityId = deserializeEntityId(jsonObject); + final DittoRuntimeException error = + GlobalErrorRegistry.getInstance().parse(errorJson, dittoHeaders); + return new StreamingSubscriptionFailed(subscriptionId, entityId, error, dittoHeaders); + }); + } + + /** + * Get the cause of the failure. + * + * @return the error in JSON format. + */ + public DittoRuntimeException getError() { + return error; + } + + /** + * Create a copy of this event with a new error. + * + * @param error the new error. + * @return the copied event with new error. + */ + public StreamingSubscriptionFailed setError(final DittoRuntimeException error) { + return new StreamingSubscriptionFailed(getSubscriptionId(), getEntityId(), error, getDittoHeaders()); + } + + @Override + public JsonPointer getResourcePath() { + return JsonPointer.empty(); + } + + @Override + public StreamingSubscriptionFailed setDittoHeaders(final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionFailed(getSubscriptionId(), getEntityId(), error, dittoHeaders); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder) { + jsonObjectBuilder.set(JsonFields.ERROR, error.toJson()); + } + + @Override + public boolean equals(final Object o) { + return super.equals(o) && Objects.equals(error, ((StreamingSubscriptionFailed) o).error); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), error); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + ", error=" + error + "]"; + } + + /** + * Json fields of this event. + */ + public static final class JsonFields { + + /** + * Json fields for a JSON representation of the error. + */ + public static final JsonFieldDefinition ERROR = + JsonFactory.newJsonObjectFieldDefinition("error"); + + JsonFields() { + throw new AssertionError(); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNext.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNext.java new file mode 100755 index 00000000000..c8539110fb9 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNext.java @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableEvent; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.json.JsonValue; + +/** + * This event is emitted after the next items to stream are ready. + * Corresponds to the reactive-streams signal {@code Subscriber#onNext(T)}. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableEvent(name = StreamingSubscriptionHasNext.NAME, typePrefix = StreamingSubscriptionEvent.TYPE_PREFIX) +public final class StreamingSubscriptionHasNext + extends AbstractStreamingSubscriptionEvent { + + /** + * Name of the event. + */ + public static final String NAME = "next"; + + /** + * Type of this event. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + private final JsonValue item; + + private StreamingSubscriptionHasNext(final String subscriptionId, + final EntityId entityId, + final JsonValue item, + final DittoHeaders dittoHeaders) { + super(TYPE, subscriptionId, entityId, dittoHeaders); + this.item = item; + } + + /** + * Constructs a new {@code SubscriptionHasNext} object. + * + * @param subscriptionId the subscription ID. + * @param entityId the entity ID of this streaming subscription event. + * @param item the "next" item. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the SubscriptionHasNext created. + * @throws NullPointerException if either argument is null. + */ + public static StreamingSubscriptionHasNext of(final String subscriptionId, + final EntityId entityId, + final JsonValue item, + final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionHasNext(subscriptionId, entityId, item, dittoHeaders); + } + + /** + * Creates a new {@code SubscriptionHasNext} from a JSON object. + * + * @param jsonObject the JSON object from which a new SubscriptionHasNext instance is to be created. + * @param dittoHeaders the headers of the command which was the cause of this event. + * @return the {@code SubscriptionHasNext} which was created from the given JSON object. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static StreamingSubscriptionHasNext fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new EventJsonDeserializer(TYPE, jsonObject) + .deserialize((revision, timestamp, metadata) -> { + final String subscriptionId = + jsonObject.getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID); + final EntityId entityId = deserializeEntityId(jsonObject); + final JsonValue item = jsonObject.getValueOrThrow(JsonFields.ITEM); + return new StreamingSubscriptionHasNext(subscriptionId, entityId, item, dittoHeaders); + }); + } + + /** + * Get the "next" item. + * + * @return the next item. + */ + public JsonValue getItem() { + return item; + } + + /** + * Create a copy of this event with a new item. + * + * @param item the new item. + * @return the copied event with new item. + */ + public StreamingSubscriptionHasNext setItem(final JsonValue item) { + return new StreamingSubscriptionHasNext(getSubscriptionId(), getEntityId(), item, getDittoHeaders()); + } + + @Override + public JsonPointer getResourcePath() { + return JsonPointer.empty(); + } + + @Override + public StreamingSubscriptionHasNext setDittoHeaders(final DittoHeaders dittoHeaders) { + return new StreamingSubscriptionHasNext(getSubscriptionId(), getEntityId(), item, dittoHeaders); + } + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder) { + jsonObjectBuilder.set(JsonFields.ITEM, item); + } + + @Override + public boolean equals(final Object o) { + // super.equals(o) guarantees getClass() == o.getClass() + return super.equals(o) && Objects.equals(item, ((StreamingSubscriptionHasNext) o).item); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), item); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + super.toString() + ", item=" + item + "]"; + } + + /** + * Json fields of this event. + */ + public static final class JsonFields { + + /** + * Json field for "next" item. + */ + public static final JsonFieldDefinition ITEM = + JsonFactory.newJsonValueFieldDefinition("item"); + + JsonFields() { + throw new AssertionError(); + } + } +} diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/package-info.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/package-info.java new file mode 100755 index 00000000000..7a13cf46a93 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/events/streaming/package-info.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.eclipse.ditto.utils.jsr305.annotations.AllValuesAreNonnullByDefault +package org.eclipse.ditto.base.model.signals.events.streaming; + diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java index 5c53e597a8c..6685dd2873e 100755 --- a/base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java @@ -19,11 +19,13 @@ import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; import java.time.Duration; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -129,6 +131,12 @@ public final class ImmutableDittoHeadersTest { .build()) .build()) .build(); + private static final Long KNOWN_AT_HISTORICAL_REVISION = 42L; + private static final Instant KNOWN_AT_HISTORICAL_TIMESTAMP = Instant.now(); + + private static final JsonObject KNOWN_HISTORICAL_HEADERS = JsonObject.newBuilder() + .set(DittoHeaderDefinition.ORIGINATOR.getKey(), "foo:bar") + .build(); static { @@ -199,6 +207,9 @@ public void settingAllKnownHeadersWorksAsExpected() { .putHeader(DittoHeaderDefinition.GET_METADATA.getKey(), KNOWN_DITTO_GET_METADATA ) .putHeader(DittoHeaderDefinition.DELETE_METADATA.getKey(), KNOWN_DITTO_DELETE_METADATA ) .putHeader(DittoHeaderDefinition.DITTO_METADATA.getKey(), KNOWN_DITTO_METADATA.formatAsString()) + .putHeader(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION)) + .putHeader(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP)) + .putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString()) .build(); assertThat(underTest).isEqualTo(expectedHeaderMap); @@ -526,6 +537,9 @@ public void toJsonReturnsExpected() { .set(DittoHeaderDefinition.GET_METADATA.getKey(), KNOWN_DITTO_GET_METADATA) .set(DittoHeaderDefinition.DELETE_METADATA.getKey(), KNOWN_DITTO_DELETE_METADATA) .set(DittoHeaderDefinition.DITTO_METADATA.getKey(), KNOWN_DITTO_METADATA) + .set(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), KNOWN_AT_HISTORICAL_REVISION) + .set(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), KNOWN_AT_HISTORICAL_TIMESTAMP.toString()) + .set(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS) .build(); final Map allKnownHeaders = createMapContainingAllKnownHeaders(); @@ -710,7 +724,7 @@ public void preserveCapitalizationOfCorrelationId() { } private static Map createMapContainingAllKnownHeaders() { - final Map result = new HashMap<>(); + final Map result = new LinkedHashMap<>(); result.put(DittoHeaderDefinition.AUTHORIZATION_CONTEXT.getKey(), AUTH_CONTEXT.toJsonString()); result.put(DittoHeaderDefinition.CORRELATION_ID.getKey(), KNOWN_CORRELATION_ID); result.put(DittoHeaderDefinition.SCHEMA_VERSION.getKey(), KNOWN_SCHEMA_VERSION.toString()); @@ -762,6 +776,9 @@ private static Map createMapContainingAllKnownHeaders() { result.put(DittoHeaderDefinition.GET_METADATA.getKey(), KNOWN_DITTO_GET_METADATA); result.put(DittoHeaderDefinition.DELETE_METADATA.getKey(), KNOWN_DITTO_DELETE_METADATA); result.put(DittoHeaderDefinition.DITTO_METADATA.getKey(), KNOWN_DITTO_METADATA.formatAsString()); + result.put(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION)); + result.put(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP)); + result.put(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString()); return result; } diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/ShardedMessageEnvelopeTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/ShardedMessageEnvelopeTest.java index ccc35d53b85..b1e2ad591f2 100644 --- a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/ShardedMessageEnvelopeTest.java +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/ShardedMessageEnvelopeTest.java @@ -14,11 +14,12 @@ import static org.assertj.core.api.Assertions.assertThat; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.entity.type.EntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObject; import org.junit.Test; import nl.jqno.equalsverifier.EqualsVerifier; @@ -31,7 +32,7 @@ public final class ShardedMessageEnvelopeTest { private static final DittoHeaders DITTO_HEADERS = DittoHeaders.empty(); private static final EntityId MESSAGE_ID = - EntityId.of(EntityType.of("thing"), "org.eclipse.ditto.test:thingId"); + NamespacedEntityId.of(EntityType.of("thing"), "org.eclipse.ditto.test:thingId"); private static final String TYPE = "message-type"; private static final JsonObject MESSAGE = JsonFactory.newObjectBuilder().set("hello", "world").build(); diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscriptionTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscriptionTest.java new file mode 100755 index 00000000000..520e43134e2 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/CancelStreamingSubscriptionTest.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonPointer; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link CancelStreamingSubscription}. + */ +public final class CancelStreamingSubscriptionTest { + + @Test + public void assertImmutability() { + assertInstancesOf(CancelStreamingSubscription.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(CancelStreamingSubscription.class) + .withRedefinedSuperclass() + .verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final CancelStreamingSubscription underTest = CancelStreamingSubscription.of( + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), + JsonPointer.of("/"), + UUID.randomUUID().toString(), dittoHeaders); + final CancelStreamingSubscription deserialized = CancelStreamingSubscription.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } + +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscriptionTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscriptionTest.java new file mode 100755 index 00000000000..1c0c8f31988 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/RequestFromStreamingSubscriptionTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonPointer; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link RequestFromStreamingSubscription}. + */ +public final class RequestFromStreamingSubscriptionTest { + + @Test + public void assertImmutability() { + assertInstancesOf(RequestFromStreamingSubscription.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(RequestFromStreamingSubscription.class) + .withRedefinedSuperclass() + .verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final RequestFromStreamingSubscription + underTest = RequestFromStreamingSubscription.of(NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), + JsonPointer.of("/"), UUID.randomUUID().toString(), 9L, dittoHeaders); + final RequestFromStreamingSubscription deserialized = RequestFromStreamingSubscription.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } + +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEventsTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEventsTest.java new file mode 100755 index 00000000000..c78c8dab609 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEventsTest.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.commands.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.AllowedReason.provided; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.time.Instant; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonPointer; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link SubscribeForPersistedEvents}. + */ +public final class SubscribeForPersistedEventsTest { + + private static final String KNOWN_ENTITY_ID_STR = "foo:bar"; + private static final String KNOWN_ENTITY_TYPE_STR = "thing"; + private static final String KNOWN_RESOURCE_PATH = "/"; + + private static final long KNOWN_FROM_REV = 23L; + private static final long KNOWN_TO_REV = 42L; + private static final String KNOWN_FROM_TS = "2022-10-25T14:00:00Z"; + private static final String KNOWN_TO_TS = "2022-10-25T15:00:00Z"; + + private static final String JSON_ALL_FIELDS = JsonFactory.newObjectBuilder() + .set(Command.JsonFields.TYPE, SubscribeForPersistedEvents.TYPE) + .set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_TYPE, KNOWN_ENTITY_TYPE_STR) + .set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_ID, KNOWN_ENTITY_ID_STR) + .set(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH, KNOWN_RESOURCE_PATH) + .set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_REVISION, KNOWN_FROM_REV) + .set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_REVISION, KNOWN_TO_REV) + .set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP, KNOWN_FROM_TS) + .set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_TIMESTAMP, KNOWN_TO_TS) + .build() + .toString(); + + private static final String JSON_MINIMAL = JsonFactory.newObjectBuilder() + .set(Command.JsonFields.TYPE, SubscribeForPersistedEvents.TYPE) + .set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_TYPE, KNOWN_ENTITY_TYPE_STR) + .set(StreamingSubscriptionCommand.JsonFields.JSON_ENTITY_ID, KNOWN_ENTITY_ID_STR) + .set(StreamingSubscriptionCommand.JsonFields.JSON_RESOURCE_PATH, KNOWN_RESOURCE_PATH) + .set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_REVISION, KNOWN_FROM_REV) + .set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_REVISION, KNOWN_TO_REV) + .build().toString(); + + @Test + public void assertImmutability() { + assertInstancesOf(SubscribeForPersistedEvents.class, + areImmutable(), + provided(Instant.class).isAlsoImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(SubscribeForPersistedEvents.class) + .withRedefinedSuperclass() + .verify(); + } + + @Test + public void toJsonWithAllFieldsSet() { + final SubscribeForPersistedEvents command = SubscribeForPersistedEvents.of( + NamespacedEntityId.of(EntityType.of(KNOWN_ENTITY_TYPE_STR), KNOWN_ENTITY_ID_STR), + JsonPointer.of(KNOWN_RESOURCE_PATH), + KNOWN_FROM_REV, + KNOWN_TO_REV, + Instant.parse(KNOWN_FROM_TS), + Instant.parse(KNOWN_TO_TS), + DittoHeaders.empty() + ); + + final String json = command.toJsonString(); + assertThat(json).isEqualTo(JSON_ALL_FIELDS); + } + + @Test + public void toJsonWithOnlyRequiredFieldsSet() { + final SubscribeForPersistedEvents command = SubscribeForPersistedEvents.of( + NamespacedEntityId.of(EntityType.of(KNOWN_ENTITY_TYPE_STR), KNOWN_ENTITY_ID_STR), + JsonPointer.of(KNOWN_RESOURCE_PATH), + KNOWN_FROM_REV, + KNOWN_TO_REV, + DittoHeaders.empty()); + final String json = command.toJsonString(); + assertThat(json).isEqualTo(JSON_MINIMAL); + } + + @Test + public void fromJsonWithAllFieldsSet() { + final SubscribeForPersistedEvents command = SubscribeForPersistedEvents.of( + NamespacedEntityId.of(EntityType.of(KNOWN_ENTITY_TYPE_STR), KNOWN_ENTITY_ID_STR), + JsonPointer.of(KNOWN_RESOURCE_PATH), + KNOWN_FROM_REV, + KNOWN_TO_REV, + Instant.parse(KNOWN_FROM_TS), + Instant.parse(KNOWN_TO_TS), + DittoHeaders.empty() + ); + assertThat(SubscribeForPersistedEvents.fromJson(JsonObject.of(JSON_ALL_FIELDS), DittoHeaders.empty())) + .isEqualTo(command); + } + + @Test + public void fromJsonWithOnlyRequiredFieldsSet() { + assertThat(SubscribeForPersistedEvents.fromJson(JsonObject.of(JSON_MINIMAL), DittoHeaders.empty())) + .isEqualTo(SubscribeForPersistedEvents.of( + NamespacedEntityId.of(EntityType.of(KNOWN_ENTITY_TYPE_STR), KNOWN_ENTITY_ID_STR), + JsonPointer.of(KNOWN_RESOURCE_PATH), + KNOWN_FROM_REV, + KNOWN_TO_REV, + DittoHeaders.empty())); + } + +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCompleteTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCompleteTest.java new file mode 100644 index 00000000000..1f628cf250f --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCompleteTest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link StreamingSubscriptionComplete}. + */ +public final class StreamingSubscriptionCompleteTest { + + @Test + public void assertImmutability() { + assertInstancesOf(StreamingSubscriptionComplete.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(StreamingSubscriptionComplete.class).withRedefinedSuperclass().verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final StreamingSubscriptionComplete underTest = StreamingSubscriptionComplete.of(UUID.randomUUID().toString(), + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), dittoHeaders); + final StreamingSubscriptionComplete deserialized = StreamingSubscriptionComplete.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreatedTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreatedTest.java new file mode 100644 index 00000000000..cc97fc9cee3 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionCreatedTest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link StreamingSubscriptionCreated}. + */ +public final class StreamingSubscriptionCreatedTest { + + @Test + public void assertImmutability() { + assertInstancesOf(StreamingSubscriptionCreated.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(StreamingSubscriptionCreated.class).withRedefinedSuperclass().verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final StreamingSubscriptionCreated underTest = StreamingSubscriptionCreated.of(UUID.randomUUID().toString(), + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), dittoHeaders); + final StreamingSubscriptionCreated deserialized = StreamingSubscriptionCreated.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailedTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailedTest.java new file mode 100644 index 00000000000..03ccc64ab3b --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionFailedTest.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.AllowedReason.provided; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.InvalidRqlExpressionException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.GlobalErrorRegistry; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link StreamingSubscriptionFailed}. + */ +public final class StreamingSubscriptionFailedTest { + + @Test + public void assertImmutability() { + assertInstancesOf(StreamingSubscriptionFailed.class, areImmutable(), + provided(DittoRuntimeException.class).isAlsoImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(StreamingSubscriptionFailed.class).withRedefinedSuperclass().verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final DittoRuntimeException error = GlobalErrorRegistry.getInstance() + .parse(InvalidRqlExpressionException.newBuilder().build().toJson(), dittoHeaders); + final StreamingSubscriptionFailed underTest = StreamingSubscriptionFailed.of(UUID.randomUUID().toString(), + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), error, dittoHeaders); + final StreamingSubscriptionFailed deserialized = StreamingSubscriptionFailed.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } +} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNextTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNextTest.java new file mode 100644 index 00000000000..15c371b4567 --- /dev/null +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/signals/events/streaming/StreamingSubscriptionHasNextTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.signals.events.streaming; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mutabilitydetector.unittesting.AllowedReason.provided; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.UUID; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonArray; +import org.eclipse.ditto.json.JsonValue; +import org.junit.Test; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Tests {@link StreamingSubscriptionHasNext}. + */ +public final class StreamingSubscriptionHasNextTest { + + @Test + public void assertImmutability() { + assertInstancesOf(StreamingSubscriptionHasNext.class, areImmutable(), provided(JsonValue.class).isAlsoImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(StreamingSubscriptionHasNext.class).withRedefinedSuperclass().verify(); + } + + @Test + public void serialization() { + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder().randomCorrelationId().build(); + final JsonArray items = JsonArray.of("[{\"x\":1},{\"x\":2}]"); + final StreamingSubscriptionHasNext + underTest = StreamingSubscriptionHasNext.of(UUID.randomUUID().toString(), + NamespacedEntityId.of(EntityType.of("thing"), "foo:bar"), items, dittoHeaders); + final StreamingSubscriptionHasNext deserialized = StreamingSubscriptionHasNext.fromJson(underTest.toJson(), dittoHeaders); + assertThat(deserialized).isEqualTo(underTest); + } +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionHistoryNotAccessibleException.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionHistoryNotAccessibleException.java new file mode 100755 index 00000000000..e51a650b3ae --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionHistoryNotAccessibleException.java @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model.signals.commands.exceptions; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.net.URI; +import java.text.MessageFormat; +import java.time.Instant; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.ConnectivityException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Thrown if historical data of the Connection was either not present in Ditto at all or if the requester had insufficient + * permissions to access it. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableException(errorCode = ConnectionHistoryNotAccessibleException.ERROR_CODE) +public final class ConnectionHistoryNotAccessibleException extends DittoRuntimeException + implements ConnectivityException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "connection.history.notfound"; + + private static final String MESSAGE_TEMPLATE = + "The Connection with ID ''{0}'' at revision ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String MESSAGE_TEMPLATE_TS = + "The Connection with ID ''{0}'' at timestamp ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String DEFAULT_DESCRIPTION = + "Check if the ID of your requested Connection was correct, you have sufficient permissions and ensure that the " + + "asked for revision/timestamp does not exceed the history-retention-duration."; + + private static final long serialVersionUID = -998877665544332221L; + + private ConnectionHistoryNotAccessibleException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + super(ERROR_CODE, HttpStatus.NOT_FOUND, dittoHeaders, message, description, cause, href); + } + + private static String getMessage(final ConnectionId connectionId, final long revision) { + checkNotNull(connectionId, "connectionId"); + return MessageFormat.format(MESSAGE_TEMPLATE, String.valueOf(connectionId), String.valueOf(revision)); + } + + private static String getMessage(final ConnectionId connectionId, final Instant timestamp) { + checkNotNull(connectionId, "connectionId"); + checkNotNull(timestamp, "timestamp"); + return MessageFormat.format(MESSAGE_TEMPLATE_TS, String.valueOf(connectionId), timestamp.toString()); + } + + /** + * A mutable builder for a {@code ConnectionHistoryNotAccessibleException}. + * + * @param connectionId the ID of the connection. + * @param revision the asked for revision of the connection. + * @return the builder. + * @throws NullPointerException if {@code connectionId} is {@code null}. + */ + public static Builder newBuilder(final ConnectionId connectionId, final long revision) { + return new Builder(connectionId, revision); + } + + /** + * A mutable builder for a {@code ConnectionHistoryNotAccessibleException}. + * + * @param connectionId the ID of the connection. + * @param timestamp the asked for timestamp of the connection. + * @return the builder. + * @throws NullPointerException if {@code connectionId} is {@code null}. + */ + public static Builder newBuilder(final ConnectionId connectionId, final Instant timestamp) { + return new Builder(connectionId, timestamp); + } + + /** + * Constructs a new {@code ConnectionHistoryNotAccessibleException} object with given message. + * + * @param message detail message. This message can be later retrieved by the {@link #getMessage()} method. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new ConnectionHistoryNotAccessibleException. + * @throws NullPointerException if {@code dittoHeaders} is {@code null}. + */ + public static ConnectionHistoryNotAccessibleException fromMessage(@Nullable final String message, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromMessage(message, dittoHeaders, new Builder()); + } + + /** + * Constructs a new {@code ConnectionHistoryNotAccessibleException} object with the exception message extracted from the given + * JSON object. + * + * @param jsonObject the JSON to read the {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new ConnectionHistoryNotAccessibleException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static ConnectionHistoryNotAccessibleException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link ConnectionHistoryNotAccessibleException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() { + description(DEFAULT_DESCRIPTION); + } + + private Builder(final ConnectionId connectionId, final long revision) { + this(); + message(ConnectionHistoryNotAccessibleException.getMessage(connectionId, revision)); + } + + private Builder(final ConnectionId connectionId, final Instant timestamp) { + this(); + message(ConnectionHistoryNotAccessibleException.getMessage(connectionId, timestamp)); + } + + @Override + protected ConnectionHistoryNotAccessibleException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new ConnectionHistoryNotAccessibleException(dittoHeaders, message, description, cause, href); + } + + } + +} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/ConnectivityRootActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/ConnectivityRootActor.java index 899a587fd66..9a3357892a9 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/ConnectivityRootActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/ConnectivityRootActor.java @@ -83,8 +83,6 @@ private ConnectivityRootActor(final ConnectivityConfig connectivityConfig, final var dittoExtensionsConfig = ScopedConfig.dittoExtension(actorSystem.settings().config()); final var enforcerActorPropsFactory = ConnectionEnforcerActorPropsFactory.get(actorSystem, dittoExtensionsConfig); - final var connectionSupervisorProps = - ConnectionSupervisorActor.props(commandForwarder, pubSubMediator, enforcerActorPropsFactory); // Create persistence streaming actor (with no cache) and make it known to pubSubMediator. final ActorRef persistenceStreamingActor = startChildActor(ConnectionPersistenceStreamingActorCreator.ACTOR_NAME, @@ -100,6 +98,10 @@ private ConnectivityRootActor(final ConnectivityConfig connectivityConfig, DittoProtocolSub.get(actorSystem); final MongoReadJournal mongoReadJournal = MongoReadJournal.newInstance(actorSystem); + + final var connectionSupervisorProps = + ConnectionSupervisorActor.props(commandForwarder, pubSubMediator, enforcerActorPropsFactory, + mongoReadJournal); startClusterSingletonActor( PersistencePingActor.props( startConnectionShardRegion(actorSystem, connectionSupervisorProps, clusterConfig), diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/ConnectionConfig.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/ConnectionConfig.java index 04c61b58f74..b23934471b1 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/ConnectionConfig.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/ConnectionConfig.java @@ -20,15 +20,18 @@ import org.eclipse.ditto.base.service.config.supervision.WithSupervisorConfig; import org.eclipse.ditto.edge.service.acknowledgements.AcknowledgementConfig; import org.eclipse.ditto.internal.utils.config.KnownConfigValue; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithActivityCheckConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithSnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.WithCleanupConfig; /** * Provides configuration settings for Connectivity service's connection behaviour. */ @Immutable -public interface ConnectionConfig extends WithSupervisorConfig, WithActivityCheckConfig, WithCleanupConfig { +public interface ConnectionConfig extends WithSupervisorConfig, WithActivityCheckConfig, WithCleanupConfig, + WithSnapshotConfig { /** * Returns the amount of time for how long the connection actor waits for response from client actors. @@ -75,6 +78,13 @@ public interface ConnectionConfig extends WithSupervisorConfig, WithActivityChec */ SnapshotConfig getSnapshotConfig(); + /** + * Returns the config of the connection event journal behaviour. + * + * @return the config. + */ + EventConfig getEventConfig(); + /** * Returns the maximum number of Targets within a connection. * diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/DefaultConnectionConfig.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/DefaultConnectionConfig.java index b30128e9885..f06c6971097 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/DefaultConnectionConfig.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/DefaultConnectionConfig.java @@ -25,7 +25,9 @@ import org.eclipse.ditto.internal.utils.config.ConfigWithFallback; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultActivityCheckConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultEventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultSnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.CleanupConfig; @@ -47,6 +49,7 @@ public final class DefaultConnectionConfig implements ConnectionConfig { private final String blockedHostRegex; private final SupervisorConfig supervisorConfig; private final SnapshotConfig snapshotConfig; + private final EventConfig eventConfig; private final DefaultAcknowledgementConfig acknowledgementConfig; private final CleanupConfig cleanupConfig; private final Amqp10Config amqp10Config; @@ -74,6 +77,7 @@ private DefaultConnectionConfig(final ConfigWithFallback config) { blockedHostRegex = config.getString(ConnectionConfigValue.BLOCKED_HOST_REGEX.getConfigPath()); supervisorConfig = DefaultSupervisorConfig.of(config); snapshotConfig = DefaultSnapshotConfig.of(config); + eventConfig = DefaultEventConfig.of(config); acknowledgementConfig = DefaultAcknowledgementConfig.of(config); cleanupConfig = CleanupConfig.of(config); amqp10Config = DefaultAmqp10Config.of(config); @@ -153,6 +157,11 @@ public SnapshotConfig getSnapshotConfig() { return snapshotConfig; } + @Override + public EventConfig getEventConfig() { + return eventConfig; + } + @Override public Integer getMaxNumberOfTargets() { return maxNumberOfTargets; @@ -246,6 +255,7 @@ public boolean equals(final Object o) { Objects.equals(blockedHostRegex, that.blockedHostRegex) && Objects.equals(supervisorConfig, that.supervisorConfig) && Objects.equals(snapshotConfig, that.snapshotConfig) && + Objects.equals(eventConfig, that.eventConfig) && Objects.equals(acknowledgementConfig, that.acknowledgementConfig) && Objects.equals(cleanupConfig, that.cleanupConfig) && Objects.equals(amqp10Config, that.amqp10Config) && @@ -266,7 +276,7 @@ public boolean equals(final Object o) { @Override public int hashCode() { return Objects.hash(clientActorAskTimeout, clientActorRestartsBeforeEscalation, allowedHostnames, - blockedHostnames, blockedSubnets, blockedHostRegex, supervisorConfig, snapshotConfig, + blockedHostnames, blockedSubnets, blockedHostRegex, supervisorConfig, snapshotConfig, eventConfig, acknowledgementConfig, cleanupConfig, maxNumberOfTargets, maxNumberOfSources, activityCheckConfig, fieldsEncryptionConfig, amqp10Config, amqp091Config, mqttConfig, kafkaConfig, httpPushConfig, ackLabelDeclareInterval, priorityUpdateInterval, shutdownTimeout, allClientActorsOnOneNode); @@ -283,6 +293,7 @@ public String toString() { ", blockedHostRegex=" + blockedHostRegex + ", supervisorConfig=" + supervisorConfig + ", snapshotConfig=" + snapshotConfig + + ", eventConfig=" + eventConfig + ", acknowledgementConfig=" + acknowledgementConfig + ", cleanUpConfig=" + cleanupConfig + ", amqp10Config=" + amqp10Config + diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseClientActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseClientActor.java index dddec5cbe04..ea9a47e3c2c 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseClientActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BaseClientActor.java @@ -61,7 +61,9 @@ import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; import org.eclipse.ditto.base.model.signals.WithType; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; import org.eclipse.ditto.connectivity.api.BaseClientState; import org.eclipse.ditto.connectivity.api.InboundSignal; import org.eclipse.ditto.connectivity.api.OutboundSignal; @@ -114,6 +116,7 @@ import org.eclipse.ditto.connectivity.service.util.ConnectionPubSub; import org.eclipse.ditto.connectivity.service.util.ConnectivityMdcEntryKey; import org.eclipse.ditto.edge.service.headers.DittoHeadersValidator; +import org.eclipse.ditto.edge.service.streaming.StreamingSubscriptionManager; import org.eclipse.ditto.internal.models.signal.correlation.MatchingValidationResult; import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory; import org.eclipse.ditto.internal.utils.akka.logging.ThreadSafeDittoLoggingAdapter; @@ -178,8 +181,10 @@ public abstract class BaseClientActor extends AbstractFSMWithStash inboundMappingSink; private ActorRef outboundDispatchingActor; private ActorRef subscriptionManager; + private ActorRef streamingSubscriptionManager; private ActorRef tunnelActor; private int ackregatorCount = 0; private boolean shuttingDown = false; @@ -325,6 +331,7 @@ protected void init() { inboundMappingSink = getInboundMappingSink(protocolAdapter, inboundDispatchingSink); subscriptionManager = startSubscriptionManager(commandForwarderActorSelection, connectivityConfig().getClientConfig()); + streamingSubscriptionManager = startStreamingSubscriptionManager(commandForwarderActorSelection, connectivityConfig().getClientConfig()); if (connection.getSshTunnel().map(SshTunnel::isEnabled).orElse(false)) { tunnelActor = childActorNanny.startChildActor(SshTunnelActor.ACTOR_NAME, @@ -505,6 +512,7 @@ protected void cleanupFurtherResourcesOnConnectionTimeout(final BaseClientState */ protected FSMStateFunctionBuilder inAnyState() { return matchEvent(RetrieveConnectionMetrics.class, (command, data) -> retrieveConnectionMetrics(command)) + .event(StreamingSubscriptionCommand.class, this::forwardStreamingSubscriptionCommand) .event(ThingSearchCommand.class, this::forwardThingSearchCommand) .event(RetrieveConnectionStatus.class, this::retrieveConnectionStatus) .event(ResetConnectionMetrics.class, this::resetConnectionMetrics) @@ -1716,6 +1724,8 @@ private FSM.State handleInboundSignal(final Inb final Signal signal = inboundSignal.getSignal(); if (signal instanceof WithSubscriptionId) { dispatchSearchCommand((WithSubscriptionId) signal); + } else if (signal instanceof WithStreamingSubscriptionId) { + dispatchStreamingSubscriptionCommand((WithStreamingSubscriptionId) signal); } else { final var entityId = tryExtractEntityId(signal).orElseThrow(); connectionPubSub.publishSignal(inboundSignal.asDispatched(), connectionId(), entityId, getSender()); @@ -1746,6 +1756,24 @@ private void dispatchSearchCommand(final WithSubscriptionId searchCommand) { } } + private void dispatchStreamingSubscriptionCommand( + final WithStreamingSubscriptionId streamingSubscriptionCommand) { + + final String subscriptionId = streamingSubscriptionCommand.getSubscriptionId(); + if (subscriptionId.length() > subscriptionIdPrefixLength) { + final var prefix = subscriptionId.substring(0, subscriptionIdPrefixLength); + connectionPubSub.publishSignal(streamingSubscriptionCommand, connectionId(), prefix, ActorRef.noSender()); + } else { + // command is invalid or outdated, dropping. + logger.withCorrelationId(streamingSubscriptionCommand) + .info("Dropping streaming subscription command with invalid subscription ID: <{}>", + streamingSubscriptionCommand); + connectionLogger.failure(InfoProviderFactory.forSignal(streamingSubscriptionCommand), + "Dropping streaming subscription command with invalid subscription ID: " + + streamingSubscriptionCommand.getSubscriptionId()); + } + } + private Instant getInConnectionStatusSince() { return stateData().getInConnectionStatusSince(); } @@ -1891,6 +1919,19 @@ private ActorRef startSubscriptionManager(final ActorSelection proxyActor, final return getContext().actorOf(props, SubscriptionManager.ACTOR_NAME); } + /** + * Start the streaming subscription manager. Requires MessageMappingProcessorActor to be started to work. + * Creates an actor materializer. + * + * @return reference of the streaming subscription manager. + */ + private ActorRef startStreamingSubscriptionManager(final ActorSelection proxyActor, final ClientConfig clientConfig) { + final var mat = Materializer.createMaterializer(this::getContext); + final var props = StreamingSubscriptionManager.props(clientConfig.getSubscriptionManagerTimeout(), + proxyActor, mat); + return getContext().actorOf(props, StreamingSubscriptionManager.ACTOR_NAME); + } + private FSM.State forwardThingSearchCommand(final WithDittoHeaders command, final BaseClientData data) { // Tell subscriptionManager to send search events to messageMappingProcessorActor. @@ -1906,6 +1947,19 @@ private FSM.State forwardThingSearchCommand(fin return stay(); } + private FSM.State forwardStreamingSubscriptionCommand( + final StreamingSubscriptionCommand command, + final BaseClientData data) { + // Tell subscriptionManager to send streaming subscription events to messageMappingProcessorActor. + if (stateName() == CONNECTED) { + streamingSubscriptionManager.tell(command, outboundDispatchingActor); + } else { + logger.withCorrelationId(command) + .debug("Client state <{}> is not CONNECTED; dropping <{}>", stateName(), command); + } + return stay(); + } + private FSM.State resubscribe(final Control trigger, final BaseClientData data) { subscribeAndDeclareAcknowledgementLabels(dryRun, true); startSubscriptionRefreshTimer(); diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BasePublisherActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BasePublisherActor.java index 960583434c8..9583bd4e155 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BasePublisherActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/BasePublisherActor.java @@ -48,6 +48,7 @@ import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.model.signals.acks.Acknowledgements; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.connectivity.api.ExternalMessage; import org.eclipse.ditto.connectivity.api.OutboundSignal; import org.eclipse.ditto.connectivity.model.Connection; @@ -356,7 +357,7 @@ private Stream sendMappedOutboundSignal(final OutboundSignal.M private Optional getSendingContext(final OutboundSignal.Mapped mappedOutboundSignal) { final Optional result; - if (isResponseOrErrorOrSearchEvent(mappedOutboundSignal)) { + if (isResponseOrErrorOrStreamingEvent(mappedOutboundSignal)) { final Signal source = mappedOutboundSignal.getSource(); final DittoHeaders dittoHeaders = source.getDittoHeaders(); result = dittoHeaders.getReplyTarget() @@ -369,17 +370,19 @@ private Optional getSendingContext(final OutboundSignal.Mapped m } /** - * Checks whether the passed in {@code outboundSignal} is a response or an error or a search event. + * Checks whether the passed in {@code outboundSignal} is a response or an error or a streaming event + * (including search events). * Those messages are supposed to be published at the reply target of the source whence the original command came. * * @param outboundSignal the OutboundSignal to check. * @return {@code true} if the OutboundSignal is a response or an error, {@code false} otherwise */ - private static boolean isResponseOrErrorOrSearchEvent(final OutboundSignal.Mapped outboundSignal) { + private static boolean isResponseOrErrorOrStreamingEvent(final OutboundSignal.Mapped outboundSignal) { final ExternalMessage externalMessage = outboundSignal.getExternalMessage(); return externalMessage.isResponse() || externalMessage.isError() || - outboundSignal.getSource() instanceof SubscriptionEvent; + outboundSignal.getSource() instanceof SubscriptionEvent || + outboundSignal.getSource() instanceof StreamingSubscriptionEvent; } private Optional getReplyTargetByIndex(final int replyTargetIndex) { diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/InboundDispatchingSink.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/InboundDispatchingSink.java index afe0da9db33..53d2ae0f277 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/InboundDispatchingSink.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/InboundDispatchingSink.java @@ -44,11 +44,13 @@ import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.model.signals.acks.Acknowledgements; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; import org.eclipse.ditto.base.model.signals.commands.ErrorResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.connectivity.api.ExternalMessage; import org.eclipse.ditto.connectivity.api.InboundSignal; import org.eclipse.ditto.connectivity.api.MappedInboundExternalMessage; @@ -508,6 +510,8 @@ private PartialFunction, Stream> dispatchResponsesAndS ) .match(CreateSubscription.class, cmd -> forwardToConnectionActor(cmd, sender)) .match(WithSubscriptionId.class, cmd -> forwardToClientActor(cmd, sender)) + .match(SubscribeForPersistedEvents.class, cmd -> forwardToConnectionActor(cmd, sender)) + .match(WithStreamingSubscriptionId.class, cmd -> forwardToClientActor(cmd, sender)) .matchAny(baseSignal -> ackregatorStarter.preprocess(baseSignal, (signal, isAckRequesting) -> Stream.of(new IncomingSignal(signal, getReturnAddress(sender, isAckRequesting, signal), @@ -659,7 +663,7 @@ private Stream forwardToClientActor(final Signal signal, @Nullable fin return Stream.empty(); } - private Stream forwardToConnectionActor(final CreateSubscription command, @Nullable final ActorRef sender) { + private Stream forwardToConnectionActor(final Command command, @Nullable final ActorRef sender) { connectionActor.tell(command, sender); return Stream.empty(); diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundDispatchingActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundDispatchingActor.java index 630147a10f0..3e63e41d785 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundDispatchingActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/OutboundDispatchingActor.java @@ -31,6 +31,7 @@ import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.connectivity.api.InboundSignal; import org.eclipse.ditto.connectivity.api.OutboundSignalFactory; import org.eclipse.ditto.connectivity.model.Target; @@ -79,6 +80,7 @@ public Receive createReceive() { .match(InboundSignal.class, this::inboundSignal) .match(CommandResponse.class, this::forwardWithoutCheck) .match(SubscriptionEvent.class, this::forwardWithoutCheck) + .match(StreamingSubscriptionEvent.class, this::forwardWithoutCheck) .match(DittoRuntimeException.class, this::forwardWithoutCheck) .match(Signal.class, this::handleSignal) .matchAny(message -> logger.warning("Unknown message: <{}>", message)) diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActor.java index ebed209c9ef..40411357804 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActor.java @@ -45,6 +45,7 @@ import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.connectivity.api.BaseClientState; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionId; @@ -55,6 +56,7 @@ import org.eclipse.ditto.connectivity.model.ConnectivityStatus; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommandInterceptor; import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionFailedException; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionHistoryNotAccessibleException; import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CheckConnectionLogsActive; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CloseConnection; @@ -198,12 +200,13 @@ public final class ConnectionPersistenceActor @Nullable private Instant recoveredAt; ConnectionPersistenceActor(final ConnectionId connectionId, + final MongoReadJournal mongoReadJournal, final ActorRef commandForwarderActor, final ActorRef pubSubMediator, final Trilean allClientActorsOnOneNode, final Config connectivityConfigOverwrites) { - super(connectionId); + super(connectionId, mongoReadJournal); this.actorSystem = context().system(); cluster = Cluster.get(actorSystem); final Config dittoExtensionConfig = ScopedConfig.dittoExtension(actorSystem.settings().config()); @@ -263,18 +266,21 @@ protected DittoDiagnosticLoggingAdapter createLogger() { * Creates Akka configuration object for this actor. * * @param connectionId the connection ID. - * @param commandForwarderActor the actor used to send signals into the ditto cluster.. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the connection. + * @param commandForwarderActor the actor used to send signals into the ditto cluster. + * @param pubSubMediator pub-sub-mediator for the shutdown behavior. * @param pubSubMediator the pubSubMediator * @param connectivityConfigOverwrites the overwrites for the connectivity config for the given connection. * @return the Akka configuration Props object. */ public static Props props(final ConnectionId connectionId, + final MongoReadJournal mongoReadJournal, final ActorRef commandForwarderActor, final ActorRef pubSubMediator, final Config connectivityConfigOverwrites ) { - return Props.create(ConnectionPersistenceActor.class, connectionId, - commandForwarderActor, pubSubMediator, Trilean.UNKNOWN, connectivityConfigOverwrites); + return Props.create(ConnectionPersistenceActor.class, connectionId, mongoReadJournal, + commandForwarderActor, pubSubMediator,Trilean.UNKNOWN, connectivityConfigOverwrites); } /** @@ -350,6 +356,16 @@ protected DittoRuntimeExceptionBuilder newNotAccessibleExceptionBuilder() { return ConnectionNotAccessibleException.newBuilder(entityId); } + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final long revision) { + return ConnectionHistoryNotAccessibleException.newBuilder(entityId, revision); + } + + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final Instant timestamp) { + return ConnectionHistoryNotAccessibleException.newBuilder(entityId, timestamp); + } + @Override protected void publishEvent(@Nullable final Connection previousEntity, final ConnectivityEvent event) { if (event instanceof ConnectionDeleted) { @@ -665,6 +681,9 @@ protected Receive matchAnyAfterInitialization() { // CreateSubscription is a ThingSearchCommand, but it is created in InboundDispatchingSink from an // adaptable and directly sent to this actor: .match(CreateSubscription.class, this::startThingSearchSession) + // SubscribeForPersistedEvents is created in InboundDispatchingSink from an + // adaptable and directly sent to this actor: + .match(SubscribeForPersistedEvents.class, this::startStreamingSubscriptionSession) .matchEquals(Control.CHECK_LOGGING_ACTIVE, this::checkLoggingEnabled) .matchEquals(Control.TRIGGER_UPDATE_PRIORITY, this::triggerUpdatePriority) .match(UpdatePriority.class, this::updatePriority) @@ -725,6 +744,17 @@ private void startThingSearchSession(final CreateSubscription command) { augmentWithPrefixAndForward(command, entity.getClientCount()); } + private void startStreamingSubscriptionSession(final SubscribeForPersistedEvents command) { + if (entity == null) { + logDroppedSignal(command, command.getType(), "No Connection configuration available."); + return; + } + log.debug("Forwarding <{}> to client actors.", command); + // compute the next prefix according to subscriptionCounter and the currently configured client actor count + // ignore any "prefix" field from the command + augmentWithPrefixAndForward(command, entity.getClientCount()); + } + private void augmentWithPrefixAndForward(final CreateSubscription createSubscription, final int clientCount) { subscriptionCounter = (subscriptionCounter + 1) % Math.max(1, clientCount); final var prefix = getPrefix(getSubscriptionPrefixLength(clientCount), subscriptionCounter); @@ -732,6 +762,14 @@ private void augmentWithPrefixAndForward(final CreateSubscription createSubscrip connectionPubSub.publishSignal(commandWithPrefix, entityId, prefix, ActorRef.noSender()); } + private void augmentWithPrefixAndForward(final SubscribeForPersistedEvents subscribeForPersistedEvents, + final int clientCount) { + subscriptionCounter = (subscriptionCounter + 1) % Math.max(1, clientCount); + final var prefix = getPrefix(getSubscriptionPrefixLength(clientCount), subscriptionCounter); + final var commandWithPrefix = subscribeForPersistedEvents.setPrefix(prefix); + connectionPubSub.publishSignal(commandWithPrefix, entityId, prefix, ActorRef.noSender()); + } + private static String getPrefix(final int prefixLength, final int subscriptionCounter) { final var prefixPattern = MessageFormat.format("%0{0,number}X", prefixLength); return String.format(prefixPattern, subscriptionCounter); diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionSupervisorActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionSupervisorActor.java index 05d69b8560b..4e443542bc9 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionSupervisorActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionSupervisorActor.java @@ -41,6 +41,7 @@ import org.eclipse.ditto.connectivity.service.config.DittoConnectivityConfig; import org.eclipse.ditto.connectivity.service.enforcement.ConnectionEnforcerActorPropsFactory; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import com.typesafe.config.Config; @@ -95,10 +96,12 @@ public final class ConnectionSupervisorActor private final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory; @SuppressWarnings("unused") - private ConnectionSupervisorActor(final ActorRef commandForwarderActor, final ActorRef pubSubMediator, - final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory) { + private ConnectionSupervisorActor(final ActorRef commandForwarderActor, + final ActorRef pubSubMediator, + final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory, + final MongoReadJournal mongoReadJournal) { - super(null, CONNECTIVITY_DEFAULT_LOCAL_ASK_TIMEOUT); + super(null, mongoReadJournal, CONNECTIVITY_DEFAULT_LOCAL_ASK_TIMEOUT); this.commandForwarderActor = commandForwarderActor; this.pubSubMediator = pubSubMediator; this.enforcerActorPropsFactory = enforcerActorPropsFactory; @@ -114,14 +117,16 @@ private ConnectionSupervisorActor(final ActorRef commandForwarderActor, final Ac * @param commandForwarder the actor used to send signals into the ditto cluster. * @param pubSubMediator pub-sub-mediator for the shutdown behavior. * @param enforcerActorPropsFactory used to create the enforcer actor. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the connection. * @return the {@link Props} to create this actor. */ public static Props props(final ActorRef commandForwarder, final ActorRef pubSubMediator, - final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory) { + final ConnectionEnforcerActorPropsFactory enforcerActorPropsFactory, + final MongoReadJournal mongoReadJournal) { return Props.create(ConnectionSupervisorActor.class, commandForwarder, pubSubMediator, - enforcerActorPropsFactory); + enforcerActorPropsFactory, mongoReadJournal); } @Override @@ -183,7 +188,7 @@ protected void handleMessagesDuringStartup(final Object message) { @Override protected Props getPersistenceActorProps(final ConnectionId entityId) { - return ConnectionPersistenceActor.props(entityId, commandForwarderActor, pubSubMediator, + return ConnectionPersistenceActor.props(entityId, mongoReadJournal, commandForwarderActor, pubSubMediator, connectivityConfigOverwrites); } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectivityMongoEventAdapter.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectivityMongoEventAdapter.java index b9478f4fd4e..2d1f1465293 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectivityMongoEventAdapter.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectivityMongoEventAdapter.java @@ -12,28 +12,31 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence; +import java.util.HashMap; +import java.util.Map; + import akka.actor.ExtendedActorSystem; import org.eclipse.ditto.base.model.signals.JsonParsable; +import org.eclipse.ditto.base.model.signals.events.Event; import org.eclipse.ditto.base.model.signals.events.EventJsonDeserializer; import org.eclipse.ditto.base.model.signals.events.EventRegistry; import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry; +import org.eclipse.ditto.base.service.config.DittoServiceConfig; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionModified; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; -import org.eclipse.ditto.connectivity.service.config.DittoConnectivityConfig; -import org.eclipse.ditto.connectivity.service.config.FieldsEncryptionConfig; +import org.eclipse.ditto.connectivity.service.config.DefaultConnectionConfig; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; +import org.eclipse.ditto.connectivity.service.config.DittoConnectivityConfig; +import org.eclipse.ditto.connectivity.service.config.FieldsEncryptionConfig; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; -import java.util.HashMap; -import java.util.Map; - /** * EventAdapter for {@link ConnectivityEvent}s persisted into * akka-persistence event-journal. Converts Events to MongoDB BSON objects and vice versa. @@ -43,8 +46,10 @@ public final class ConnectivityMongoEventAdapter extends AbstractMongoEventAdapt private final FieldsEncryptionConfig encryptionConfig; private final Logger logger; - public ConnectivityMongoEventAdapter(@Nullable final ExtendedActorSystem system) { - super(system, createEventRegistry()); + public ConnectivityMongoEventAdapter(final ExtendedActorSystem system) { + super(system, createEventRegistry(), DefaultConnectionConfig.of( + DittoServiceConfig.of(DefaultScopedConfig.dittoScoped(system.settings().config()), "connectivity")) + .getEventConfig()); logger = LoggerFactory.getLogger(ConnectivityMongoEventAdapter.class); final DittoConnectivityConfig connectivityConfig = DittoConnectivityConfig.of( DefaultScopedConfig.dittoScoped(system.settings().config())); @@ -56,11 +61,14 @@ public ConnectivityMongoEventAdapter(@Nullable final ExtendedActorSystem system) } @Override - protected JsonObject performToJournalMigration(final JsonObject jsonObject) { + protected JsonObjectBuilder performToJournalMigration(final Event event, final JsonObject jsonObject) { if (encryptionConfig.isEncryptionEnabled()) { - return JsonFieldsEncryptor.encrypt(jsonObject, ConnectivityConstants.ENTITY_TYPE.toString(), encryptionConfig.getJsonPointers(), encryptionConfig.getSymmetricalKey()); + final JsonObject superObject = super.performToJournalMigration(event, jsonObject).build(); + return JsonFieldsEncryptor.encrypt(superObject, ConnectivityConstants.ENTITY_TYPE.toString(), + encryptionConfig.getJsonPointers(), encryptionConfig.getSymmetricalKey()) + .toBuilder(); } - return jsonObject; + return super.performToJournalMigration(event, jsonObject); } @Override diff --git a/connectivity/service/src/main/resources/connectivity.conf b/connectivity/service/src/main/resources/connectivity.conf index 6c58dc8e0ee..353972cdc5f 100644 --- a/connectivity/service/src/main/resources/connectivity.conf +++ b/connectivity/service/src/main/resources/connectivity.conf @@ -208,8 +208,23 @@ ditto { } snapshot { - threshold = 10 + # the interval when to do snapshot for a Connection which had changes to it interval = 15m + interval = ${?CONNECTION_SNAPSHOT_INTERVAL} # may be overridden with this environment variable + + # the threshold after how many changes to a Connection to do a snapshot + threshold = 10 + threshold = ${?CONNECTION_SNAPSHOT_THRESHOLD} # may be overridden with this environment variable + } + + event { + # define the DittoHeaders to persist when persisting events to the journal + # those can e.g. be retrieved as additional "audit log" information when accessing a historical connection revision + historical-headers-to-persist = [ + #"ditto-originator" # who (user-subject/connection-pre-auth-subject) issued the event + #"correlation-id" + ] + historical-headers-to-persist = ${?CONNECTION_EVENT_HISTORICAL_HEADERS_TO_PERSIST} } activity-check { @@ -677,27 +692,62 @@ ditto { } cleanup { + # enabled configures whether background cleanup is enabled or not + # If enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process enabled = true enabled = ${?CLEANUP_ENABLED} + # history-retention-duration configures the duration of how long to "keep" events and snapshots before being + # allowed to remove them in scope of cleanup. + # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read + # journal. + history-retention-duration = 0d + history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} + + # quiet-period defines how long to stay in a state where the background cleanup is not yet started + # Applies after: + # - starting the service + # - each "completed" background cleanup run (all entities were cleaned up) quiet-period = 5m quiet-period = ${?CLEANUP_QUIET_PERIOD} + # interval configures how often a "credit decision" is made. + # The background cleanup works with a credit system and does only generate new "cleanup credits" if the MongoDB + # currently has capacity to do cleanups. interval = 60s interval = ${?CLEANUP_INTERVAL} + # timer-threshold configures the maximum database latency to give out credit for cleanup actions. + # If write operations to the MongoDB within the last `interval` had a `max` value greater to the configured + # threshold, no new cleanup credits will be issued for the next `interval`. + # Which throttles cleanup when MongoDB is currently under heavy (write) load. timer-threshold = 150ms timer-threshold = ${?CLEANUP_TIMER_THRESHOLD} + # credits-per-batch configures how many "cleanup credits" should be generated per `interval` as long as the + # write operations to the MongoDB are less than the configured `timer-threshold`. + # Limits the rate of cleanup actions to this many per credit decision interval. + # One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`. credits-per-batch = 1 credits-per-batch = ${?CLEANUP_CREDITS_PER_BATCH} + # reads-per-query configures the number of snapshots to scan per MongoDB query. + # Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned + # with the amount of "cleanup credits" issued per `interval` - in order to avoid long running queries. reads-per-query = 100 reads-per-query = ${?CLEANUP_READS_PER_QUERY} + # writes-per-credit configures the number of documents to delete for each credit. + # If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead + # to 10 delete operations performed against MongoDB. writes-per-credit = 100 writes-per-credit = ${?CLEANUP_WRITES_PER_CREDIT} + # delete-final-deleted-snapshot configures whether for a deleted entity, the final snapshot (containing the + # "deleted" information) should be deleted or not. + # If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with + # a revision number 1 higher than the previously deleted entity. If the final snapshot is deleted as well, + # recreation of an entity with the same ID will lead to revisionNumber=1 after its recreation. delete-final-deleted-snapshot = false delete-final-deleted-snapshot = ${?CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT} } @@ -1113,6 +1163,18 @@ akka-contrib-mongodb-persistence-connection-journal { } } +akka-contrib-mongodb-persistence-connection-journal-read { + class = "akka.contrib.persistence.mongodb.MongoReadJournal" + plugin-dispatcher = "connection-persistence-dispatcher" + + overrides { + journal-collection = "connection_journal" + journal-index = "connection_journal_index" + realtime-collection = "connection_realtime" + metadata-collection = "connection_metadata" + } +} + akka-contrib-mongodb-persistence-connection-snapshots { class = "akka.contrib.persistence.mongodb.MongoSnapshots" plugin-dispatcher = "connection-persistence-dispatcher" diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalCommandRegistryTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalCommandRegistryTest.java index 8feb00887e4..827d15af927 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalCommandRegistryTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalCommandRegistryTest.java @@ -17,6 +17,7 @@ import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoAddConnectionLogEntry; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionIdsByTag; @@ -66,7 +67,8 @@ public ConnectivityServiceGlobalCommandRegistryTest() { PurgeEntities.class, ModifySplitBrainResolver.class, PublishSignal.class, - SudoAddConnectionLogEntry.class + SudoAddConnectionLogEntry.class, + SubscribeForPersistedEvents.class ); } diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalEventRegistryTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalEventRegistryTest.java index 4df61bcefb2..ae6b9142a42 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalEventRegistryTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalEventRegistryTest.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.connectivity.service; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.internal.utils.persistentactors.EmptyEvent; import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases; @@ -31,7 +32,8 @@ public ConnectivityServiceGlobalEventRegistryTest() { SubscriptionCreated.class, ThingsOutOfSync.class, ThingSnapshotTaken.class, - EmptyEvent.class + EmptyEvent.class, + StreamingSubscriptionComplete.class ); } diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/TestConstants.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/TestConstants.java index b0a787054d4..821780b6797 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/TestConstants.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/TestConstants.java @@ -110,6 +110,7 @@ import org.eclipse.ditto.internal.utils.cluster.DistPubSubAccess; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.config.ScopedConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.config.PingConfig; import org.eclipse.ditto.internal.utils.protocol.config.ProtocolConfig; import org.eclipse.ditto.internal.utils.pubsub.StreamingType; @@ -924,7 +925,8 @@ public static ActorRef createConnectionSupervisorActor(final ConnectionId connec final var enforcerActorPropsFactory = ConnectionEnforcerActorPropsFactory.get(actorSystem, dittoExtensionsConfig); final Props props = - ConnectionSupervisorActor.props(commandForwarderActor, pubSubMediator, enforcerActorPropsFactory); + ConnectionSupervisorActor.props(commandForwarderActor, pubSubMediator, enforcerActorPropsFactory, + Mockito.mock(MongoReadJournal.class)); final Props shardRegionMockProps = Props.create(ShardRegionMockActor.class, props, connectionId.toString()); diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java index e268f81ff06..8362c2e16b5 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java @@ -82,6 +82,7 @@ import org.eclipse.ditto.internal.utils.akka.ActorSystemResource; import org.eclipse.ditto.internal.utils.akka.PingCommand; import org.eclipse.ditto.internal.utils.akka.controlflow.WithSender; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import org.eclipse.ditto.internal.utils.test.Retry; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; @@ -91,6 +92,7 @@ import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; +import org.mockito.Mockito; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigValueFactory; @@ -248,6 +250,7 @@ public TestActor.AutoPilot run(final ActorRef sender, final Object msg) { final var connectionActorProps = Props.create(ConnectionPersistenceActor.class, () -> new ConnectionPersistenceActor(connectionId, + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), Trilean.TRUE, @@ -893,14 +896,16 @@ public void recoverDeletedConnection() { @Test public void exceptionDuringClientActorPropsCreation() { - final var connectionActorProps = ConnectionPersistenceActor.props( - TestConstants.createRandomConnectionId(), commandForwarderActor, pubSubMediatorProbe.ref(), - ConfigFactory.empty() - ); - final var testProbe = actorSystemResource1.newTestProbe(); final var supervisor = actorSystemResource1.newTestProbe(); - final var connectionActorRef = supervisor.childActorOf(connectionActorProps); + + final var connectionActorProps = ConnectionPersistenceActor.props( + TestConstants.createRandomConnectionId(), + Mockito.mock(MongoReadJournal.class), + commandForwarderActor, + pubSubMediatorProbe.ref(), + ConfigFactory.empty()); + final var connectionActorRef = supervisor.childActorOf(connectionActorProps, "connection"); // create connection final CreateConnection createConnection = createConnection(); @@ -921,6 +926,7 @@ public void exceptionDuringClientActorPropsCreation() { @Test public void exceptionDueToCustomValidator() { final var connectionActorProps = ConnectionPersistenceActor.props(TestConstants.createRandomConnectionId(), + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), ConfigFactory.empty()); @@ -1155,6 +1161,7 @@ public TestActor.AutoPilot run(final ActorRef sender, final Object msg) { final var testProbe = actorSystemResource1.newTestProbe(); final var connectionActorProps = Props.create(ConnectionPersistenceActor.class, () -> new ConnectionPersistenceActor(connectionId, + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), Trilean.TRUE, @@ -1217,6 +1224,7 @@ public void retriesStartingClientActor() { Props.create( ConnectionPersistenceActor.class, () -> new ConnectionPersistenceActor(connectionId, + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), Trilean.FALSE, @@ -1244,6 +1252,7 @@ public void escalatesWhenClientActorFailsTooOften() { Props.create( ConnectionPersistenceActor.class, () -> new ConnectionPersistenceActor(connectionId, + Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), Trilean.FALSE, diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceOperationsActorIT.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceOperationsActorIT.java index 883749f2701..af6b7f9171e 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceOperationsActorIT.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceOperationsActorIT.java @@ -34,10 +34,12 @@ import org.eclipse.ditto.connectivity.service.enforcement.ConnectionEnforcerActorPropsFactory; import org.eclipse.ditto.internal.utils.config.ScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.ops.eventsource.MongoEventSourceITAssertions; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.utils.jsr305.annotations.AllValuesAreNonnullByDefault; import org.junit.ClassRule; import org.junit.Test; +import org.mockito.Mockito; import com.typesafe.config.Config; @@ -134,7 +136,8 @@ protected ActorRef startEntityActor(final ActorSystem system, final ActorRef pub final var dittoExtensionsConfig = ScopedConfig.dittoExtension(system.settings().config()); final var enforcerActorPropsFactory = ConnectionEnforcerActorPropsFactory.get(system, dittoExtensionsConfig); final Props props = - ConnectionSupervisorActor.props(proxyActorProbe.ref(), pubSubMediator, enforcerActorPropsFactory); + ConnectionSupervisorActor.props(proxyActorProbe.ref(), pubSubMediator, enforcerActorPropsFactory, + Mockito.mock(MongoReadJournal.class)); return system.actorOf(props, String.valueOf(id)); } diff --git a/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml b/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml index 4ac16530a28..e114dd11074 100644 --- a/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml +++ b/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml @@ -263,6 +263,9 @@ entries: - title: Search url: /basic-search.html output: web + - title: History capabilities + url: /basic-history.html + output: web - title: Acknowledgements / QoS url: /basic-acknowledgements.html output: web @@ -492,6 +495,10 @@ entries: url: /protocol-specification-connections-announcement.html output: web + - title: Streaming subscriptions (history) + url: /protocol-specification-streaming-subscription.html + output: web + - title: Bindings url: /protocol-bindings.html output: web diff --git a/documentation/src/main/resources/_data/tags.yml b/documentation/src/main/resources/_data/tags.yml index 84db28d555c..72d45c03e79 100644 --- a/documentation/src/main/resources/_data/tags.yml +++ b/documentation/src/main/resources/_data/tags.yml @@ -11,6 +11,7 @@ allowed-tags: - model - signal - http + - history - search - protocol - connectivity diff --git a/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-failed-payload.json b/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-failed-payload.json new file mode 100644 index 00000000000..db6bff8bc08 --- /dev/null +++ b/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-failed-payload.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "description": "", + "title": "", + "properties": { + "subscriptionId": { + "type": "string", + "description": "Identifier of the streaming subscription delivered by a previous \"created\" event." + }, + "error": { + "type": "object", + "description": "The _error_ field contains information about the error that occurred.", + "properties": { + "status": { + "type": "integer", + "description": "The status code that indicates the cause of the error. The semantics of the used status codes are based on the [HTTP status codes](https://tools.ietf.org/html/rfc7231#section-6)." + }, + "error": { + "type": "string", + "description": "The error code that uniquely identifies the error." + }, + "message": { + "type": "string", + "description": "A human readable message that explains in detail what went wrong." + }, + "description": { + "type": "string", + "description": "Contains further information about the error e.g. a hint what caused the problem and how to solve it." + } + } + } + }, + "required": ["subscriptionId", "error"] +} diff --git a/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-next-payload.json b/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-next-payload.json new file mode 100644 index 00000000000..dfc01e22393 --- /dev/null +++ b/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-next-payload.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "description": "", + "title": "", + "properties": { + "subscriptionId": { + "type": "string", + "description": "Identifier of the streaming subscription delivered by a previous \"created\" event." + }, + "item": { + "description": "A JSON value containing JSON representations of the contained item." + } + }, + "required": ["subscriptionId", "item"] +} diff --git a/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-request-payload.json b/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-request-payload.json new file mode 100644 index 00000000000..bc61bdd94e4 --- /dev/null +++ b/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-request-payload.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "description": "", + "title": "", + "properties": { + "subscriptionId": { + "type": "string", + "description": "Identifier of the streaming subscription delivered by a previous \"created\" event." + }, + "demand": { + "type": "number", + "description": "How many items to request. Must be a positive integer." + } + }, + "required": ["subscriptionId", "demand"] +} diff --git a/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-subscribe-for-persisted-events-payload.json b/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-subscribe-for-persisted-events-payload.json new file mode 100644 index 00000000000..bf525456c20 --- /dev/null +++ b/documentation/src/main/resources/jsonschema/protocol-streaming-subscription-subscribe-for-persisted-events-payload.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "description": "", + "title": "", + "properties": { + "fromHistoricalRevision": { + "type": "integer", + "description": "The revision to start the streaming from." + }, + "toHistoricalRevision": { + "type": "integer", + "description": "The revision to stop the streaming at." + }, + "fromHistoricalTimestamp": { + "type": "string", + "format": "date-time", + "description": "The timestamp to start the streaming from." + }, + "toHistoricalTimestamp": { + "type": "string", + "format": "date-time", + "description": "The timestamp to stop the streaming at." + } + } +} diff --git a/documentation/src/main/resources/jsonschema/protocol-streaming-subscriptionid.json b/documentation/src/main/resources/jsonschema/protocol-streaming-subscriptionid.json new file mode 100644 index 00000000000..92758ecd8a7 --- /dev/null +++ b/documentation/src/main/resources/jsonschema/protocol-streaming-subscriptionid.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "description": "", + "title": "", + "properties": { + "subscriptionId": { + "type": "string", + "description": "Identifier of the streaming subscription delivered by a previous \"created\" event." + } + }, + "required": ["subscriptionId"] +} diff --git a/documentation/src/main/resources/pages/ditto/basic-history.md b/documentation/src/main/resources/pages/ditto/basic-history.md new file mode 100644 index 00000000000..89c9cec0e9b --- /dev/null +++ b/documentation/src/main/resources/pages/ditto/basic-history.md @@ -0,0 +1,178 @@ +--- +title: History capabilities +keywords: history, historic, historian +tags: [history] +permalink: basic-history.html +--- + +Starting with **Eclipse Ditto 3.2.0**, APIs for retrieving the history of the following entities is provided: +* [things](basic-thing.html) +* [policies](basic-policy.html) +* [connections](basic-connections.html) + +The capabilities of these APIs are the following: + +| Entity | Retrieving entity at a specific revision or timestamp | Streaming modification events of an entity specifying from/to revision/timestamp | +|------------|-------------------------------------------------------|----------------------------------------------------------------------------------| +| Thing | ✓ | ✓ | +| Policy | ✓ | ✓ | +| Connection | ✓ | no | + +{% include note.html content="Ditto's history API capabilities are not comparable with the features of a time series database. + E.g. no aggregations on or compactions of the historical data can be done." %} + + +## Retrieving entity from history + +Provides: +* Finding out the state of an entity (thing, policy, connection) at a given: + * revision number + * timestamp +* Retrieving "historical headers" persisted together with a modification (see [configuring historical headers to persist](#configuring-historical-headers-to-persist)) + +Target use cases: +* Compare changes to entity (e.g. a connection) to a former state + * In order to solve potential errors in e.g. policy or connection configuration +* **Audit log**: Find out who (which subject) did a change to an entity + * E.g. in order to find out who changed a policy/connection + * Configure [which headers to persist as historical headers](#configuring-historical-headers-to-persist) to e.g. include the subject which did a modification + +### Retrieve entity at specific revision + +Retrieving an entity at an (historical) revision, set the header `at-historical-revision` to a `long` number for all +"retrieve" commands of persisted state. + +Example for the HTTP API: +```bash +# Access a thing: +curl -u ditto:ditto 'http://localhost:8080/api/2/things/org.eclipse.ditto:thing-1' \ + --header 'at-historical-revision: 1' + +# Access a policy: +curl -u ditto:ditto 'http://localhost:8080/api/2/policies/org.eclipse.ditto:policy-1' \ + --header 'at-historical-revision: 1' + +# Access a connection: +curl -u ditto:ditto 'http://localhost:8080/api/2/connections/some-connection-1' \ + --header 'at-historical-revision: 1' +``` + +The same functionality is available via a [header of a Ditto Protocol](protocol-specification.html#headers) message. + +If [historical headers](#configuring-historical-headers-to-persist) were configured to be persisted, they can be found +in the response header named `historical-headers`. + +### Retrieve entity at specific timestamp + +Retrieving an entity at an (historical) timestamp, set the header `at-historical-timestamp` to an ISO-8601 formatted +`string` for all "retrieve" commands of persisted state. + +Example for the HTTP API: +```bash +# Access a thing: +curl -u ditto:ditto 'http://localhost:8080/api/2/things/org.eclipse.ditto:thing-1' \ + --header 'at-historical-timestamp: 2022-10-24T03:11:15Z' + +# Access a policy: +curl -u ditto:ditto 'http://localhost:8080/api/2/policies/org.eclipse.ditto:policy-1' \ + --header 'at-historical-timestamp: 2022-10-24T06:11:15Z' + +# Access a connection: +curl -u ditto:ditto 'http://localhost:8080/api/2/connections/some-connection-1' \ + --header 'at-historical-timestamp: 2022-10-24T07:11Z' +``` + +The same functionality is available via a [header of a Ditto Protocol](protocol-specification.html#headers) message. + +If [historical headers](#configuring-historical-headers-to-persist) were configured to be persisted, they can be found +in the response header named `historical-headers`. + + +## Streaming historical events of entity + +Provides: +* A stream of changes to a specific thing or policy, based on specified: + * entity ID + * start revision number (and optional stop revision number) + * start timestamp (and optional stop timestamp) +* Retrieving "historical headers" persisted together with a modification (see [configuring historical headers to persist](#configuring-historical-headers-to-persist)) + +Target use cases: +* Inspect the changes of an entity over time + * E.g. displaying a value on a chart with that way + +### Streaming historical events via SSE + +The easiest way to stream historical events is the [SSE (Server Sent Event) API](httpapi-sse.html). +This API is however **only available for things** (not for policies). + +Use the following query parameters in order to specify the start/stop revision/timestamp. + +Either use the revision based parameters: +* `from-historical-revision`: specifies the revision number to start streaming historical modification events from +* `to-historical-revision`: optionally specifies the revision number to stop streaming at (if omitted, it streams events until the current state of the entity) + +Alternatively, use the timestamp based parameters: +* `from-historical-timestamp`: specifies the timestamp to start streaming historical modification events from +* `to-historical-timestamp`: optionally specifies the timestamp to stop streaming at (if omitted, it streams events until the current state of the entity) + +The messages sent over the SSE are the same as for the [SSE (Server Sent Event) API](httpapi-sse.html), each historical +modification event is "normalized" to the Thing JSON representation. + +Examples: +```bash +# stream complete history starting from earliest available revision of a thing: +curl --http2 -u ditto:ditto -H 'Accept:text/event-stream' -N \ + http://localhost:8080/api/2/things/org.eclipse.ditto:thing-2?from-historical-revision=0&fields=thingId,attributes,features,_revision,_modified + +# stream specific history range of a thing based on revisions: +curl --http2 -u ditto:ditto -H 'Accept:text/event-stream' -N \ + http://localhost:8080/api/2/things/org.eclipse.ditto:thing-2?from-historical-revision=23&to-historical-revision=42&fields=thingId,attributes,features,_revision,_modified + +# stream specific history range of a thing based on timestamps: +curl --http2 -u ditto:ditto -H 'Accept:text/event-stream' -N \ + http://localhost:8080/api/2/things/org.eclipse.ditto:thing-2?from-historical-timestamp=2022-10-24T11:44:36Z&to-historical-timestamp=2022-10-24T11:44:37Z&fields=thingId,attributes,features,_revision,_modified + +# stream specific history range, additionally selecting _context in "fields" which contains the historical headers: +curl --http2 -u ditto:ditto -H 'Accept:text/event-stream' -N \ + http://localhost:8080/api/2/things/org.eclipse.ditto:thing-2?from-historical-revision=0&fields=thingId,attributes,features,_revision,_modified,_context +``` + +### Streaming historical events via Ditto Protocol + +Please inspect the [protocol specification of DittoProtocol messages for streaming persisted events](protocol-specification-streaming-subscription.html) +to find out how to stream historical (persisted) events via DittoProtocol. +Using the DittoProtocol, historical events can be streamed either via WebSocket or connections. + + +## Configuring historical headers to persist + +In the configuration of the services (things, policies, connectivity) there is a section where to configure the historical +headers to persist, for example this is the section for policies: + +```hocon +event { + # define the DittoHeaders to persist when persisting events to the journal + # those can e.g. be retrieved as additional "audit log" information when accessing a historical policy revision + historical-headers-to-persist = [ + #"ditto-originator" # who (user-subject/connection-pre-auth-subject) issued the event + #"correlation-id" + ] + historical-headers-to-persist = ${?POLICY_EVENT_HISTORICAL_HEADERS_TO_PERSIST} +} +``` + +By default, no headers are persisted as historical headers, but it could e.g. make sense to persist the `ditto-originator` +in order to provide "audit log" functionality in order to find out who (which subject) changed a policy at which time. + +## Cleanup retention time configuration + +In order to be able to access the history of entities, their journal database entries must not be cleaned up too quickly. + +By default, Ditto enables the [background cleanup](installation-operating.html#managing-background-cleanup) in order to +delete "stale" (when not using the history feature) data from the MongoDB. + +If Ditto shall be used with history capabilities, the cleanup has either +* be disabled completely (which however could lead to a lot of used database storage) +* or be configured with a `history-retention-duration` of a duration how long to keep "the history" before cleaning up + snapshots and events diff --git a/documentation/src/main/resources/pages/ditto/installation-operating.md b/documentation/src/main/resources/pages/ditto/installation-operating.md index 9634b5aea47..32e847b234d 100644 --- a/documentation/src/main/resources/pages/ditto/installation-operating.md +++ b/documentation/src/main/resources/pages/ditto/installation-operating.md @@ -788,9 +788,88 @@ However, [Manage connections via Piggyback commands](connectivity-manage-connect #### Managing background cleanup -Ditto deletes unnecessary events and snapshots in the background according to database load. Each Things, Policies and -Connectivity instance has an actor coordinating a portion of the background cleanup process. The actor responds to -piggyback-commands to query its state and configuration, modify its configuration, and restart the background cleanup +Ditto deletes unnecessary events and snapshots in the background according to database load. + +##### Configuration of background cleanup + +The background cleanup configuration is available for: +* Policies: [policies.conf](https://github.com/eclipse/ditto/blob/master/policies/service/src/main/resources/policies.conf) +* Things: [things.conf](https://github.com/eclipse/ditto/blob/master/things/service/src/main/resources/things.conf) +* Connectivity: [connectivity.conf](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) + +And has the following config parameters: +```hocon +cleanup { + # enabled configures whether background cleanup is enabled or not + # If enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process + enabled = true + enabled = ${?CLEANUP_ENABLED} + + # history-retention-duration configures the duration of how long to "keep" events and snapshots before being + # allowed to remove them in scope of cleanup. + # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read + # journal. + history-retention-duration = 0d + history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} + + # quiet-period defines how long to stay in a state where the background cleanup is not yet started + # Applies after: + # - starting the service + # - each "completed" background cleanup run (all entities were cleaned up) + quiet-period = 5m + quiet-period = ${?CLEANUP_QUIET_PERIOD} + + # interval configures how often a "credit decision" is made. + # The background cleanup works with a credit system and does only generate new "cleanup credits" if the MongoDB + # currently has capacity to do cleanups. + interval = 3s + interval = ${?CLEANUP_INTERVAL} + + # timer-threshold configures the maximum database latency to give out credit for cleanup actions. + # If write operations to the MongoDB within the last `interval` had a `max` value greater to the configured + # threshold, no new cleanup credits will be issued for the next `interval`. + # Which throttles cleanup when MongoDB is currently under heavy (write) load. + timer-threshold = 150ms + timer-threshold = ${?CLEANUP_TIMER_THRESHOLD} + + # credits-per-batch configures how many "cleanup credits" should be generated per `interval` as long as the + # write operations to the MongoDB are less than the configured `timer-threshold`. + # Limits the rate of cleanup actions to this many per credit decision interval. + # One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`. + credits-per-batch = 3 + credits-per-batch = ${?CLEANUP_CREDITS_PER_BATCH} + + # reads-per-query configures the number of snapshots to scan per MongoDB query. + # Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned + # with the amount of "cleanup credits" issued per `interval` - in order to avoid long running queries. + reads-per-query = 100 + reads-per-query = ${?CLEANUP_READS_PER_QUERY} + + # writes-per-credit configures the number of documents to delete for each credit. + # If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead + # to 10 delete operations performed against MongoDB. + writes-per-credit = 100 + writes-per-credit = ${?CLEANUP_WRITES_PER_CREDIT} + + # delete-final-deleted-snapshot configures whether for a deleted entity, the final snapshot (containing the + # "deleted" information) should be deleted or not. + # If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with + # a revision number 1 higher than the previously deleted entity. If the final snapshot is deleted as well, + # recreation of an entity with the same ID will lead to revisionNumber=1 after its recreation. + delete-final-deleted-snapshot = false + delete-final-deleted-snapshot = ${?CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT} +} +``` + +By default, background cleanup is enabled for all entities and the retention duration is configured to `0d` (0 days), +meaning that no history will be kept for a longer time. + +In order to use Ditto's [history capabilities](basic-history.html), the configuration has to be adjusted accordingly. + +##### Adjustment of background cleanup during runtime + +Each Things, Policies and Connectivity instance has an actor coordinating a portion of the background cleanup process. +The actor responds to piggyback-commands to query its state and configuration, modify its configuration, and restart the background cleanup process. Each command is sent to the actor selection `/user/Root/persistenceCleanup`, where diff --git a/documentation/src/main/resources/pages/ditto/protocol-specification-streaming-subscription.md b/documentation/src/main/resources/pages/ditto/protocol-specification-streaming-subscription.md new file mode 100644 index 00000000000..714394eafc0 --- /dev/null +++ b/documentation/src/main/resources/pages/ditto/protocol-specification-streaming-subscription.md @@ -0,0 +1,168 @@ +--- +title: Streaming subscription protocol specification +keywords: protocol, specification, thing, policy, stream, subscription, history, historical +tags: [protocol, history] +permalink: protocol-specification-streaming-subscription.html +--- + +The [history capabilities](basic-history.html) of the Ditto protocol consists of 3 commands and 4 events that together +implement the [reactive-streams](https://reactive-streams.org) protocol over any duplex transport layer. +For each streaming subscription request, Ditto acts as the reactive-streams publisher of pages of historical events, +and the client acts as the subscriber. +By reactive-streams means, the client controls how fast pages are delivered to it and may cancel +a request before all results are sent. + +While [connections](basic-connections.html) do not expose or require a duplex transport layer, +the streaming subscription protocol is available for them as well: Send commands from client to Ditto via any +[connection source](basic-connections.html#sources). For each command, 0 or more events from Ditto to client +are published to the reply-target of the source. + +[ps]: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/Flow.Publisher.html#subscribe(java.util.concurrent.Flow.Subscriber) +[ss]: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/Flow.Subscriber.html#onSubscribe(java.util.concurrent.Flow.Subscription) +[sn]: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/Flow.Subscriber.html#onNext(T) +[sc]: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/Flow.Subscriber.html#onComplete() +[se]: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/Flow.Subscriber.html#onError(java.lang.Throwable) +[nr]: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/Flow.Subscription.html#request(long) +[nc]: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/Flow.Subscription.html#cancel() +[n]: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/Flow.Subscription.html + +For reactive-streams on the JVM, a publisher-subscriber pair is identified by a [Subscription][n] object according +to reference equality. +Similarly, the streaming subscription protocol commands and events of one request are identified by a subscription ID. + +Each streaming subscription protocol command or event corresponds to a reactive-streams _signal_ and are bound +by the same rules in the [reactive-streams specification](https://github.com/reactive-streams/reactive-streams-jvm/blob/v1.0.3/README.md). + +| Reactive-streams signal | Streaming subscription protocol message topic |Type | Message direction | +|------------------------------|-----------------------------------------------------------------------------------------------------------------------|-------|-------------------| +| [Publisher#subscribe][ps] | [`////streaming/subscribeForPersistedEvents`](#subscribe-for-persisted-events) |Command| Client to Ditto | +| [Subscription#request][nr] | [`////streaming/request`](#request) |Command| Client to Ditto | +| [Subscription#cancel][nc] | [`////streaming/cancel`](#cancel) |Command| Client to Ditto | +| [Subscriber#onSubscribe][ss] | [`////streaming/created`](#created) |Event | Ditto to Client | +| [Subscriber#onNext][sn] | [`////streaming/next`](#next) |Event | Ditto to Client | +| [Subscriber#onComplete][sc] | [`////streaming/complete`](#complete) |Event | Ditto to Client | +| [Subscriber#onError][se] | [`////streaming/failed`](#failed) |Event | Ditto to Client | + +## Interaction pattern + +For one request, the commands from client to Ditto should follow this protocol: +``` +subscribe request* cancel? +``` +The client should send one ["subscribeForPersistedEvents"](#subscribe-for-persisted-events) command, +followed by multiple ["request"](#request) commands and an optional ["cancel"](#cancel) command. + +In response to a ["subscribeForPersistedEvents"](#subscribe-for-persisted-events) command and after each ["request"](#request) command, +Ditto will send 0 or more events to the client according to the following protocol: +``` +created next* (complete | failed)? +``` +A ["created"](#created) event bearing the subscription ID is always sent. +0 or more ["next"](#next) events are sent according to the amount of results requested +by the client. A ["complete"](#complete) or ["failed"](#failed) event comes at the +end unless the client sends a ["cancel"](#cancel) command before the results are exhausted. + +There is no special event in response to a ["cancel"](#cancel) command. +The client may continue to receive buffered ["next"](#next), +["complete"](#complete) or ["failed"](#failed) events after sending a ["cancel"](#cancel) command. + +In addition to the rules of reactive-streams, Ditto guarantees that no ["complete"](#complete) or +["failed"](#failed) event will arrive +before the client expresses its readiness by a first ["request"](#request) command. The reason is to facilitate +concurrency at the client side. Without the extra guarantee, a multi-threaded client would have to process a +["complete"](#complete) or ["failed"](#failed) event in parallel of the preceding ["created"](#created) event. +It would put the burden of sequentialization at the client side and complicate the programming there. + +## Commands from Client to Ditto + +### Subscribe for persisted events + +Sent a ["subscribeForPersistedEvents"](#subscribe-for-persisted-events) command to Ditto to start receiving persisted events as results. +Ditto will always respond with a ["created"](#created) event. + +| Field | Value | +|------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **topic** | `////streaming/subscribeForPersistedEvents` | +| **path** | `/` | +| **value** | JSON object specifying the options how the persisted events should be selected. {% include docson.html schema="jsonschema/protocol-streaming-subscription-subscribe-for-persisted-events-payload.json" %} | + +The options where to start/stop historical persisted events from can be specified in the `value` field +of a ["subscribeForPersistedEvents"](#subscribe-for-persisted-events) command. +If no options are provided at all, the complete available history for the specified entity is streamed as a result. + +### Request + +After obtaining a subscription ID from a ["created"](#created) event, +use ["request"](#request) commands to tell Ditto how many results you are prepared to receive. +Ditto will send ["next"](#next) events until all requested results are fulfilled, +the results are exhausted, or an error occurred. + +| Field | Value | +|------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| **topic** | `////streaming/request` | +| **path** | `/` | +| **value** | JSON object specifying the demand of results. {% include docson.html schema="jsonschema/protocol-streaming-subscription-request-payload.json" %} | + +### Cancel + +After obtaining a subscription ID from a ["created"](#created) event, +use a ["cancel"](#cancel) command to stop Ditto from sending more items of the results. +Pages in-flight may yet arrive, but the client will eventually stop receiving +events of the same subscription ID. + +| Field | Value | +|------------|---------------------------------------------------------------------------------------------------------------------------| +| **topic** | `////streaming/cancel` | +| **path** | `/` | +| **value** | Identifies a streaming subscription. {% include docson.html schema="jsonschema/protocol-streaming-subscriptionid.json" %} | + +## Events from Ditto to Client + +### Created + +To any ["subscribeForPersistedEvents"](#subscribe-for-persisted-events) command, Ditto will always respond with +a ["created"](#created) event. + +| Field | Value | +|------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **topic** | `////streaming/created` | +| **path** | `/` | +| **value** | Discloses the ID of a streaming subscription which all subsequent commands should include. {% include docson.html schema="jsonschema/protocol-streaming-subscriptionid.json" %} | + +### Next + +Each ["next"](#next) event contains one item of the results. +Ditto will not send more ["next"](#next) events for a given subscription ID than the total number of items +requested by previous ["request"](#request) commands. + +| Field | Value | +|------------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| **topic** | `////streaming/next` | +| **path** | `/` | +| **value** | JSON object containing one item of the results. {% include docson.html schema="jsonschema/protocol-streaming-subscription-next-payload.json" %} | + +### Complete + +A streaming subscription ends with a ["complete"](#complete) or a ["failed"](#failed) event from Ditto, +or with a ["cancel"](#cancel) command from the client. +Ditto sends a ["complete"](#complete) event when all items of the results are delivered to the client. + +| Field | Value | +|------------|---------------------------------------------------------------------------------------------------------------------------| +| **topic** | `////streaming/complete` | +| **path** | `/` | +| **value** | Identifies a streaming subscription. {% include docson.html schema="jsonschema/protocol-streaming-subscriptionid.json" %} | + +### Failed + +A streaming subscription ends with a ["complete"](#complete) or a ["failed"](#failed) event from Ditto, +or with an ["cancel"](#cancel) command from the client. +Ditto sends a ["failed"](#failed) event when an internal error occurred, +or when the client breaches the reactive-streams specification. +It is not possible to ["request"](#request) more items of the streaming subscription results after a ["failed"](#failed) event. + +| Field | Value | +|------------|------------------------------------------------------------------------------------------------------------------------------------------------------| +| **topic** | `////streaming/failed` | +| **path** | `/` | +| **value** | JSON object containing the reason for the failure. {% include docson.html schema="jsonschema/protocol-streaming-subscription-failed-payload.json" %} | diff --git a/documentation/src/main/resources/pages/ditto/protocol-specification.md b/documentation/src/main/resources/pages/ditto/protocol-specification.md index 5876b07752b..28cc5266416 100644 --- a/documentation/src/main/resources/pages/ditto/protocol-specification.md +++ b/documentation/src/main/resources/pages/ditto/protocol-specification.md @@ -86,22 +86,25 @@ since they are themselves not case-sensitive. There are some pre-defined headers, which have a special meaning for Ditto: -| Header Key | Description | Possible values | -|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| -| `content-type` | The content-type which describes the [value](#value) of Ditto Protocol messages. | `String` | -| `correlation-id` | Used for correlating protocol messages (e.g. a **command** would have the same correlation-id as the sent back **response** message). | `String` | -| `ditto-originator` | Contains the first authorization subject of the command that caused the sending of this message. Set by Ditto. | `String` | -| `if-match` | Has the same semantics as defined for the [HTTP API](httpapi-concepts.html#conditional-requests). | `String` | -| `if-none-match` | Has the same semantics as defined for the [HTTP API](httpapi-concepts.html#conditional-requests). | `String` | -| `response-required` | Configures for a **command** whether or not a **response** should be sent back. | `Boolean` - default: `true` | -| `requested-acks` | Defines which [acknowledgements](basic-acknowledgements.html) are requested for a command processed by Ditto. | `JsonArray` of `String` - default: `["twin-persisted"]` | -| `ditto-weak-ack` | Marks [weak acknowledgements](basic-acknowledgements.html) issued by Ditto. | `Boolean` - default: `false` | -| `timeout` | Defines how long the Ditto server should wait, e.g. applied when waiting for requested acknowledgements. | `String` - e.g.: `42s` or `250ms` or `1m` - default: `60s` | -| `version` | Determines in which schema version the `payload` should be interpreted. | `Number` - currently possible: \[2\] - default: `2` | -| `put-metadata` | Determines which Metadata information is stored in the thing. | `JsonArray` of `JsonObject`s containing [metadata](basic-metadata.html) to apply. | -| `condition` | The condition to evaluate before applying the request. | `String` containing [condition](basic-conditional-requests.html) to apply. | -| `live-channel-condition` | The condition to evaluate before retrieving thing data from the device. | `String` containing [live channel condition](basic-conditional-requests.html#live-channel-condition) to apply. | -| `live-channel-timeout-strategy` | The strategy to apply when a [live](protocol-twinlive.html#live) command was not answered by the actual device within the defined `timeout`. | `fail`: let the request fail with a 408 timeout error - `use-twin`: fall back to the twin, retrieving the persisted data. | +| Header Key | Description | Possible values | +|---------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| +| `content-type` | The content-type which describes the [value](#value) of Ditto Protocol messages. | `String` | +| `correlation-id` | Used for correlating protocol messages (e.g. a **command** would have the same correlation-id as the sent back **response** message). | `String` | +| `ditto-originator` | Contains the first authorization subject of the command that caused the sending of this message. Set by Ditto. | `String` | +| `if-match` | Has the same semantics as defined for the [HTTP API](httpapi-concepts.html#conditional-requests). | `String` | +| `if-none-match` | Has the same semantics as defined for the [HTTP API](httpapi-concepts.html#conditional-requests). | `String` | +| `response-required` | Configures for a **command** whether or not a **response** should be sent back. | `Boolean` - default: `true` | +| `requested-acks` | Defines which [acknowledgements](basic-acknowledgements.html) are requested for a command processed by Ditto. | `JsonArray` of `String` - default: `["twin-persisted"]` | +| `ditto-weak-ack` | Marks [weak acknowledgements](basic-acknowledgements.html) issued by Ditto. | `Boolean` - default: `false` | +| `timeout` | Defines how long the Ditto server should wait, e.g. applied when waiting for requested acknowledgements. | `String` - e.g.: `42s` or `250ms` or `1m` - default: `60s` | +| `version` | Determines in which schema version the `payload` should be interpreted. | `Number` - currently possible: \[2\] - default: `2` | +| `put-metadata` | Determines which Metadata information is stored in the thing. | `JsonArray` of `JsonObject`s containing [metadata](basic-metadata.html) to apply. | +| `condition` | The condition to evaluate before applying the request. | `String` containing [condition](basic-conditional-requests.html) to apply. | +| `live-channel-condition` | The condition to evaluate before retrieving thing data from the device. | `String` containing [live channel condition](basic-conditional-requests.html#live-channel-condition) to apply. | +| `live-channel-timeout-strategy` | The strategy to apply when a [live](protocol-twinlive.html#live) command was not answered by the actual device within the defined `timeout`. | `fail`: let the request fail with a 408 timeout error - `use-twin`: fall back to the twin, retrieving the persisted data. | +| `at-historical-revision` | The historical revision to retrieve an entity at, using the [history capabilities](basic-history.html). | `Number` - a long value of the revision to retrieve. | +| `at-historical-timestamp` | The historical timestamp in ISO-8601 format to retrieve an entity at, using the [history capabilities](basic-history.html). | `String` containing an ISO-8601 formatted timestamp. | +| `historical-headers` | Contains the historical header when using `at-historical-*` headers to retrieve an entity at a certain history point. | `JsonObject` of the headers which were configured to be persisted as historical headers. | Custom headers of messages through the [live channel](protocol-twinlive.html#live) are delivered verbatim. When naming custom headers, it is best to attach a prefix specific to your application, that does not conflict with Ditto or diff --git a/documentation/src/main/resources/pages/tags/tag_history.md b/documentation/src/main/resources/pages/tags/tag_history.md new file mode 100644 index 00000000000..2ead594845d --- /dev/null +++ b/documentation/src/main/resources/pages/tags/tag_history.md @@ -0,0 +1,9 @@ +--- +title: "History topics" +tagName: history +search: exclude +permalink: tag_history.html +sidebar: ditto_sidebar +folder: tags +--- +{% include taglogic.html %} diff --git a/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/EdgeCommandForwarderActor.java b/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/EdgeCommandForwarderActor.java index 6db6073e0e1..8fc2786b364 100644 --- a/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/EdgeCommandForwarderActor.java +++ b/edge/service/src/main/java/org/eclipse/ditto/edge/service/dispatching/EdgeCommandForwarderActor.java @@ -22,11 +22,13 @@ import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; import org.eclipse.ditto.base.model.signals.events.Event; import org.eclipse.ditto.base.service.signaltransformer.SignalTransformer; import org.eclipse.ditto.base.service.signaltransformer.SignalTransformers; import org.eclipse.ditto.connectivity.api.ConnectivityMessagingConstants; import org.eclipse.ditto.connectivity.api.commands.sudo.ConnectivitySudoCommand; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveAllConnectionIds; import org.eclipse.ditto.internal.utils.akka.logging.DittoDiagnosticLoggingAdapter; @@ -37,8 +39,10 @@ import org.eclipse.ditto.internal.utils.config.ScopedConfig; import org.eclipse.ditto.messages.model.signals.commands.MessageCommand; import org.eclipse.ditto.messages.model.signals.commands.MessageCommandResponse; +import org.eclipse.ditto.policies.model.PolicyConstants; import org.eclipse.ditto.policies.model.signals.commands.PolicyCommand; import org.eclipse.ditto.things.api.commands.sudo.SudoRetrieveThings; +import org.eclipse.ditto.things.model.ThingConstants; import org.eclipse.ditto.things.model.signals.commands.ThingCommand; import org.eclipse.ditto.things.model.signals.commands.ThingCommandResponse; import org.eclipse.ditto.things.model.signals.commands.query.RetrieveThings; @@ -146,6 +150,18 @@ public Receive createReceive() { .match(ConnectivitySudoCommand.class, this::forwardToConnectivity) .match(ThingSearchCommand.class, this::forwardToThingSearch) .match(ThingSearchSudoCommand.class, this::forwardToThingSearch) + .match(StreamingSubscriptionCommand.class, + src -> src.getEntityType().equals(ThingConstants.ENTITY_TYPE), + this::forwardToThings + ) + .match(StreamingSubscriptionCommand.class, + src -> src.getEntityType().equals(PolicyConstants.ENTITY_TYPE), + this::forwardToPolicies + ) + .match(StreamingSubscriptionCommand.class, + src -> src.getEntityType().equals(ConnectivityConstants.ENTITY_TYPE), + this::forwardToConnectivity + ) .match(Signal.class, this::handleUnknownSignal) .matchAny(m -> log.warning("Got unknown message: {}", m)) .build(); @@ -220,22 +236,23 @@ private void forwardToThingsAggregatorProxy(final Command command) { () -> signalTransformationCs.thenAccept(transformed -> aggregatorProxyActor.tell(transformed, sender))); } - private void forwardToPolicies(final PolicyCommand policyCommand) { + private void forwardToPolicies(final Signal policySignal) { final ActorRef sender = getSender(); - final CompletionStage> signalTransformationCs = applySignalTransformation(policyCommand, sender); - scheduleTask(policyCommand, () -> signalTransformationCs - .thenAccept(transformed -> { - final PolicyCommand transformedPolicyCommand = (PolicyCommand) transformed; - log.withCorrelationId(transformedPolicyCommand) + final CompletionStage> signalTransformationCs = applySignalTransformation(policySignal, sender); + scheduleTask(policySignal, () -> signalTransformationCs + .thenAccept(transformedSignal -> { + log.withCorrelationId(transformedSignal) .info("Forwarding policy command with ID <{}> and type <{}> to 'policies' shard region", - transformedPolicyCommand.getEntityId(), transformedPolicyCommand.getType()); + transformedSignal instanceof WithEntityId withEntityId ? withEntityId.getEntityId() : + null, + transformedSignal.getType()); - if (isIdempotent(transformedPolicyCommand)) { - askWithRetryCommandForwarder.forwardCommand(transformedPolicyCommand, + if (transformedSignal instanceof Command transformedCommand && isIdempotent(transformedCommand)) { + askWithRetryCommandForwarder.forwardCommand(transformedCommand, shardRegions.policies(), sender); } else { - shardRegions.policies().tell(transformedPolicyCommand, sender); + shardRegions.policies().tell(transformedSignal, sender); } })); } diff --git a/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionActor.java b/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionActor.java new file mode 100644 index 00000000000..5138fc8ae02 --- /dev/null +++ b/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionActor.java @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.edge.service.streaming; + +import java.time.Duration; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.commands.exceptions.StreamingSubscriptionProtocolErrorException; +import org.eclipse.ditto.base.model.signals.commands.exceptions.StreamingSubscriptionTimeoutException; +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionCreated; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionHasNext; +import org.eclipse.ditto.internal.utils.akka.actors.AbstractActorWithStashWithTimers; +import org.eclipse.ditto.internal.utils.akka.logging.DittoDiagnosticLoggingAdapter; +import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory; +import org.eclipse.ditto.json.JsonValue; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +import akka.actor.ActorRef; +import akka.actor.PoisonPill; +import akka.actor.Props; +import akka.actor.ReceiveTimeout; +import akka.japi.pf.ReceiveBuilder; + +/** + * Actor that translates streaming subscription commands into stream operations and stream signals into streaming + * subscription events. + */ +public final class StreamingSubscriptionActor extends AbstractActorWithStashWithTimers { + + /** + * Live on as zombie for a while to prevent timeout at client side + */ + private static final Duration ZOMBIE_LIFETIME = Duration.ofSeconds(10L); + + private final DittoDiagnosticLoggingAdapter log = DittoLoggerFactory.getDiagnosticLoggingAdapter(this); + + private final EntityId entityId; + private Subscription subscription; + private ActorRef sender; + private DittoHeaders dittoHeaders; + + StreamingSubscriptionActor(final Duration idleTimeout, + final EntityId entityId, + final ActorRef sender, + final DittoHeaders dittoHeaders) { + this.entityId = entityId; + this.sender = sender; + this.dittoHeaders = dittoHeaders; + getContext().setReceiveTimeout(idleTimeout); + } + + /** + * Create Props object for the StreamingSubscriptionActor. + * + * @param idleTimeout maximum lifetime while idling + * @param entityId the entity ID for which the streaming subscription is created. + * @param sender sender of the command that created this actor. + * @param dittoHeaders headers of the command that created this actor. + * @return Props for this actor. + */ + public static Props props(final Duration idleTimeout, final EntityId entityId, final ActorRef sender, + final DittoHeaders dittoHeaders) { + return Props.create(StreamingSubscriptionActor.class, idleTimeout, entityId, sender, dittoHeaders); + } + + /** + * Wrap a subscription actor as a reactive stream subscriber. + * + * @param streamingSubscriptionActor reference to the subscription actor. + * @param entityId the entity ID for which the subscriber is created. + * @return the actor presented as a reactive stream subscriber. + */ + public static Subscriber asSubscriber(final ActorRef streamingSubscriptionActor, + final EntityId entityId) { + return new StreamingSubscriberOps(streamingSubscriptionActor, entityId); + } + + @Override + public void postStop() { + if (subscription != null) { + subscription.cancel(); + } + } + + @Override + public Receive createReceive() { + return ReceiveBuilder.create() + .match(RequestFromStreamingSubscription.class, this::requestSubscription) + .match(CancelStreamingSubscription.class, this::cancelSubscription) + .match(StreamingSubscriptionHasNext.class, this::subscriptionHasNext) + .match(StreamingSubscriptionComplete.class, this::subscriptionComplete) + .match(StreamingSubscriptionFailed.class, this::subscriptionFailed) + .match(Subscription.class, this::onSubscribe) + .matchEquals(ReceiveTimeout.getInstance(), this::idleTimeout) + .matchAny(m -> log.warning("Unknown message: <{}>", m)) + .build(); + } + + private Receive createZombieBehavior() { + return ReceiveBuilder.create() + .match(RequestFromStreamingSubscription.class, requestSubscription -> { + log.withCorrelationId(requestSubscription) + .info("Rejecting RequestSubscription[demand={}] as zombie", + requestSubscription.getDemand()); + final String errorMessage = + "This subscription is considered cancelled. No more messages are processed."; + final StreamingSubscriptionFailed subscriptionFailed = StreamingSubscriptionFailed.of( + getSubscriptionId(), + entityId, + StreamingSubscriptionProtocolErrorException.newBuilder() + .message(errorMessage) + .build(), + requestSubscription.getDittoHeaders() + ); + getSender().tell(subscriptionFailed, ActorRef.noSender()); + }) + .matchAny(message -> log.debug("Ignoring as zombie: <{}>", message)) + .build(); + } + + private void idleTimeout(final ReceiveTimeout receiveTimeout) { + // usually a user error + log.info("Stopping due to idle timeout"); + getContext().cancelReceiveTimeout(); + final String subscriptionId = getSubscriptionId(); + final StreamingSubscriptionTimeoutException error = StreamingSubscriptionTimeoutException + .of(subscriptionId, dittoHeaders); + final StreamingSubscriptionFailed subscriptionFailed = StreamingSubscriptionFailed + .of(subscriptionId, entityId, error, dittoHeaders); + if (subscription == null) { + sender.tell(getSubscriptionCreated(), ActorRef.noSender()); + } + sender.tell(subscriptionFailed, ActorRef.noSender()); + becomeZombie(); + } + + private void onSubscribe(final Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + } else { + this.subscription = subscription; + sender.tell(getSubscriptionCreated(), ActorRef.noSender()); + unstashAll(); + } + } + + private StreamingSubscriptionCreated getSubscriptionCreated() { + return StreamingSubscriptionCreated.of(getSubscriptionId(), entityId, dittoHeaders); + } + + private void setSenderAndDittoHeaders(final StreamingSubscriptionCommand command) { + sender = getSender(); + dittoHeaders = command.getDittoHeaders(); + } + + private void requestSubscription(final RequestFromStreamingSubscription requestFromStreamingSubscription) { + if (subscription == null) { + log.withCorrelationId(requestFromStreamingSubscription).debug("Stashing <{}>", requestFromStreamingSubscription); + stash(); + } else { + log.withCorrelationId(requestFromStreamingSubscription).debug("Processing <{}>", requestFromStreamingSubscription); + setSenderAndDittoHeaders(requestFromStreamingSubscription); + subscription.request(requestFromStreamingSubscription.getDemand()); + } + } + + private void cancelSubscription(final CancelStreamingSubscription cancelStreamingSubscription) { + if (subscription == null) { + log.withCorrelationId(cancelStreamingSubscription).info("Stashing <{}>", cancelStreamingSubscription); + stash(); + } else { + log.withCorrelationId(cancelStreamingSubscription).info("Processing <{}>", cancelStreamingSubscription); + setSenderAndDittoHeaders(cancelStreamingSubscription); + subscription.cancel(); + becomeZombie(); + } + } + + private void subscriptionHasNext(final StreamingSubscriptionHasNext event) { + log.debug("Forwarding {}", event); + sender.tell(event.setDittoHeaders(dittoHeaders), ActorRef.noSender()); + } + + private void subscriptionComplete(final StreamingSubscriptionComplete event) { + // just in case: if error overtakes subscription, then there *will* be a subscription. + if (subscription == null) { + log.withCorrelationId(event).debug("Stashing <{}>", event); + stash(); + } else { + log.info("{}", event); + sender.tell(event.setDittoHeaders(dittoHeaders), ActorRef.noSender()); + becomeZombie(); + } + } + + private void subscriptionFailed(final StreamingSubscriptionFailed event) { + // just in case: if error overtakes subscription, then there *will* be a subscription. + if (subscription == null) { + log.withCorrelationId(event).debug("Stashing <{}>", event); + stash(); + } else { + // log at INFO level because user errors may cause subscription failure. + log.withCorrelationId(event).info("{}", event); + sender.tell(event.setDittoHeaders(dittoHeaders), ActorRef.noSender()); + becomeZombie(); + } + } + + private void becomeZombie() { + getTimers().startSingleTimer(PoisonPill.getInstance(), PoisonPill.getInstance(), ZOMBIE_LIFETIME); + getContext().become(createZombieBehavior()); + } + + private String getSubscriptionId() { + return getSelf().path().name(); + } + + private static final class StreamingSubscriberOps implements Subscriber { + + private final ActorRef streamingSubscriptionActor; + private final String subscriptionId; + private final EntityId entityId; + + private StreamingSubscriberOps(final ActorRef streamingSubscriptionActor, final EntityId entityId) { + this.streamingSubscriptionActor = streamingSubscriptionActor; + subscriptionId = streamingSubscriptionActor.path().name(); + this.entityId = entityId; + } + + @Override + public void onSubscribe(final Subscription subscription) { + streamingSubscriptionActor.tell(subscription, ActorRef.noSender()); + } + + @Override + public void onNext(final JsonValue item) { + final StreamingSubscriptionHasNext event = StreamingSubscriptionHasNext + .of(subscriptionId, entityId, item, DittoHeaders.empty()); + streamingSubscriptionActor.tell(event, ActorRef.noSender()); + } + + @Override + public void onError(final Throwable t) { + final StreamingSubscriptionFailed event = + StreamingSubscriptionFailed.of(subscriptionId, + entityId, + DittoRuntimeException.asDittoRuntimeException(t, e -> { + if (e instanceof IllegalArgumentException) { + // incorrect protocol from the client side + return StreamingSubscriptionProtocolErrorException.of(e, DittoHeaders.empty()); + } else { + return DittoInternalErrorException.newBuilder().cause(e).build(); + } + }), + DittoHeaders.empty()); + streamingSubscriptionActor.tell(event, ActorRef.noSender()); + } + + @Override + public void onComplete() { + final StreamingSubscriptionComplete event = StreamingSubscriptionComplete.of(subscriptionId, entityId, + DittoHeaders.empty()); + streamingSubscriptionActor.tell(event, ActorRef.noSender()); + } + } +} diff --git a/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionManager.java b/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionManager.java new file mode 100644 index 00000000000..f21b6afdccb --- /dev/null +++ b/edge/service/src/main/java/org/eclipse/ditto/edge/service/streaming/StreamingSubscriptionManager.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.edge.service.streaming; + +import java.time.Duration; +import java.util.Optional; + +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.json.Jsonifiable; +import org.eclipse.ditto.base.model.signals.FeatureToggle; +import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.base.model.signals.commands.exceptions.StreamingSubscriptionNotFoundException; +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.internal.utils.akka.logging.DittoDiagnosticLoggingAdapter; +import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory; +import org.eclipse.ditto.json.JsonValue; +import org.eclipse.ditto.protocol.ProtocolFactory; +import org.eclipse.ditto.protocol.adapter.DittoProtocolAdapter; +import org.reactivestreams.Subscriber; + +import akka.actor.AbstractActor; +import akka.actor.ActorRef; +import akka.actor.ActorSelection; +import akka.actor.Props; +import akka.japi.pf.ReceiveBuilder; +import akka.pattern.Patterns; +import akka.stream.Materializer; +import akka.stream.SourceRef; +import akka.stream.javadsl.Sink; +import akka.stream.javadsl.Source; + +/** + * Actor that manages streaming subscriptions for 1 websocket connection or 1 ConnectionPersistenceActor. + */ +public final class StreamingSubscriptionManager extends AbstractActor { + + /** + * Name of this actor. + */ + public static final String ACTOR_NAME = "streamingSubscriptionManager"; + + private static final DittoProtocolAdapter DITTO_PROTOCOL_ADAPTER = DittoProtocolAdapter.newInstance(); + private static final Duration COMMAND_FORWARDER_LOCAL_ASK_TIMEOUT = Duration.ofSeconds(15); + + private final DittoDiagnosticLoggingAdapter log = DittoLoggerFactory.getDiagnosticLoggingAdapter(this); + + private final Duration idleTimeout; + private final ActorSelection commandForwarder; + private final Materializer materializer; + + private int subscriptionIdCounter = 0; + + @SuppressWarnings("unused") + private StreamingSubscriptionManager(final Duration idleTimeout, + final ActorSelection commandForwarder, + final Materializer materializer) { + this.idleTimeout = idleTimeout; + this.commandForwarder = commandForwarder; + this.materializer = materializer; + } + + /** + * Create Props for a subscription manager. + * + * @param idleTimeout lifetime of an idle StreamingSubscriptionActor. + * @param commandForwarder recipient of streaming subscription commands. + * @param materializer materializer for the search streams. + * @return Props of the actor. + */ + public static Props props(final Duration idleTimeout, + final ActorSelection commandForwarder, + final Materializer materializer) { + + return Props.create(StreamingSubscriptionManager.class, idleTimeout, commandForwarder, materializer); + } + + @Override + public Receive createReceive() { + return ReceiveBuilder.create() + .match(RequestFromStreamingSubscription.class, this::requestSubscription) + .match(SubscribeForPersistedEvents.class, this::subscribeForPersistedEvents) + .match(CancelStreamingSubscription.class, this::cancelSubscription) + .build(); + } + + private void requestSubscription(final RequestFromStreamingSubscription requestFromStreamingSubscription) { + forwardToChild(requestFromStreamingSubscription.getSubscriptionId(), requestFromStreamingSubscription); + } + + private void cancelSubscription(final CancelStreamingSubscription cancelStreamingSubscription) { + forwardToChild(cancelStreamingSubscription.getSubscriptionId(), cancelStreamingSubscription); + } + + private void forwardToChild(final String streamingSubscriptionId, final StreamingSubscriptionCommand command) { + final Optional subscriptionActor = getContext().findChild(streamingSubscriptionId); + if (subscriptionActor.isPresent()) { + log.withCorrelationId(command).debug("Forwarding to child: <{}>", command); + subscriptionActor.get().tell(command, getSender()); + } else { + // most likely a user error + log.withCorrelationId(command) + .info("StreamingSubscriptionID not found, responding with StreamingSubscriptionFailed: <{}>", command); + final StreamingSubscriptionNotFoundException error = + StreamingSubscriptionNotFoundException.of(streamingSubscriptionId, command.getDittoHeaders()); + final StreamingSubscriptionFailed streamingSubscriptionFailed = + StreamingSubscriptionFailed.of(streamingSubscriptionId, command.getEntityId(), error, command.getDittoHeaders()); + getSender().tell(streamingSubscriptionFailed, ActorRef.noSender()); + } + } + + private void subscribeForPersistedEvents(final SubscribeForPersistedEvents subscribeForPersistedEvents) { + FeatureToggle.checkHistoricalApiAccessFeatureEnabled( + subscribeForPersistedEvents.getType(), subscribeForPersistedEvents.getDittoHeaders()); + + log.withCorrelationId(subscribeForPersistedEvents) + .info("Processing <{}>", subscribeForPersistedEvents); + final EntityId entityId = subscribeForPersistedEvents.getEntityId(); + final String subscriptionId = nextSubscriptionId(subscribeForPersistedEvents); + final Props props = StreamingSubscriptionActor.props(idleTimeout, entityId, getSender(), + subscribeForPersistedEvents.getDittoHeaders()); + final ActorRef subscriptionActor = getContext().actorOf(props, subscriptionId); + final Source itemSource = getPersistedEventsSource(subscribeForPersistedEvents); + connect(subscriptionActor, itemSource, entityId); + } + + private void connect(final ActorRef streamingSubscriptionActor, + final Source itemSource, + final EntityId entityId) { + final Subscriber subscriber = + StreamingSubscriptionActor.asSubscriber(streamingSubscriptionActor, entityId); + lazify(itemSource).runWith(Sink.fromSubscriber(subscriber), materializer); + } + + private Source getPersistedEventsSource(final SubscribeForPersistedEvents subscribe) { + + return Source.completionStageSource( + Patterns.ask(commandForwarder, subscribe, subscribe.getDittoHeaders() + .getTimeout() + .orElse(COMMAND_FORWARDER_LOCAL_ASK_TIMEOUT) + ) + .handle((response, throwable) -> { + if (response instanceof SourceRef sourceRef) { + return sourceRef.getSource() + .map(item -> { + if (item instanceof Signal signal) { + return ProtocolFactory.wrapAsJsonifiableAdaptable( + DITTO_PROTOCOL_ADAPTER.toAdaptable(signal) + ).toJson(); + } else if (item instanceof Jsonifiable jsonifiable) { + return jsonifiable.toJson(); + } else if (item instanceof JsonValue val) { + return val; + } else { + throw new IllegalStateException("Unexpected element!"); + } + }); + } else if (response instanceof DittoRuntimeException dittoRuntimeException) { + return Source.failed(dittoRuntimeException); + } else { + final var dittoRuntimeException = DittoRuntimeException + .asDittoRuntimeException(throwable, + cause -> DittoInternalErrorException.newBuilder() + .dittoHeaders(subscribe.getDittoHeaders()) + .cause(cause) + .build() + ); + return Source.failed(dittoRuntimeException); + } + }) + ); + } + + private String nextSubscriptionId(final SubscribeForPersistedEvents subscribeForPersistedEvents) { + final String prefix = subscribeForPersistedEvents.getPrefix().orElse(""); + return prefix + subscriptionIdCounter++; + } + + /** + * Make a source that never completes until downstream request. + * + * @param upstream the source to lazify. + * @param the type of elements. + * @return the lazified source. + */ + private static Source lazify(final Source upstream) { + return Source.lazySource(() -> upstream); + } + +} diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/sse/ThingsSseRouteBuilder.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/sse/ThingsSseRouteBuilder.java index 83141a79a5e..ea85d33eda3 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/sse/ThingsSseRouteBuilder.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/sse/ThingsSseRouteBuilder.java @@ -18,6 +18,7 @@ import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.time.Duration; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -41,7 +42,10 @@ import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.SignalEnrichmentFailedException; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.FeatureToggle; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.UriEncoding; import org.eclipse.ditto.gateway.service.endpoints.routes.AbstractRoute; import org.eclipse.ditto.gateway.service.endpoints.routes.things.ThingsParameter; @@ -59,8 +63,13 @@ import org.eclipse.ditto.internal.utils.metrics.instruments.counter.Counter; import org.eclipse.ditto.internal.utils.pubsub.StreamingType; import org.eclipse.ditto.internal.utils.search.SearchSource; +import org.eclipse.ditto.json.JsonCollectors; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.json.JsonFieldSelector; +import org.eclipse.ditto.json.JsonKey; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.json.JsonValue; import org.eclipse.ditto.messages.model.Message; @@ -118,10 +127,19 @@ public final class ThingsSseRouteBuilder extends RouteDirectives implements SseR private static final String LAST_EVENT_ID_HEADER = "Last-Event-ID"; private static final String PARAM_FILTER = "filter"; - private static final String PARAM_FIELDS = "fields"; + private static final String PARAM_FIELDS = ThingsParameter.FIELDS.toString(); private static final String PARAM_OPTION = "option"; private static final String PARAM_NAMESPACES = "namespaces"; private static final String PARAM_EXTRA_FIELDS = "extraFields"; + + private static final String PARAM_FROM_HISTORICAL_REVISION = "from-historical-revision"; + private static final String PARAM_TO_HISTORICAL_REVISION = "to-historical-revision"; + private static final String PARAM_FROM_HISTORICAL_TIMESTAMP = "from-historical-timestamp"; + private static final String PARAM_TO_HISTORICAL_TIMESTAMP = "to-historical-timestamp"; + + private static final JsonFieldDefinition CONTEXT = + JsonFactory.newJsonObjectFieldDefinition("_context"); + private static final PartialFunction ACCEPT_HEADER_EXTRACTOR = newAcceptHeaderExtractor(); private static final Counter THINGS_SSE_COUNTER = getCounterFor(PATH_THINGS); @@ -268,8 +286,7 @@ private Route buildThingsSseRoute(final RequestContext ctx, return createMessagesSseRoute(ctx, dhcs, thingId, jsonPointerString); } else { - params.put(ThingsParameter.FIELDS.toString(), - jsonPointerString); + params.put(PARAM_FIELDS, jsonPointerString); return createSseRoute(ctx, dhcs, JsonPointer.of(jsonPointerString), params @@ -318,13 +335,32 @@ private Route createSseRoute(final RequestContext ctx, final CompletionStage namespaces = getNamespaces(parameters.get(PARAM_NAMESPACES)); final List targetThingIds = getThingIds(parameters.get(ThingsParameter.IDS.toString())); - @Nullable final ThingFieldSelector fields = - getFieldSelector(parameters.get(ThingsParameter.FIELDS.toString())); + @Nullable final ThingFieldSelector fields = getFieldSelector(parameters.get(PARAM_FIELDS)); @Nullable final ThingFieldSelector extraFields = getFieldSelector(parameters.get(PARAM_EXTRA_FIELDS)); + + @Nullable final Long fromHistoricalRevision = Optional.ofNullable( + parameters.get(PARAM_FROM_HISTORICAL_REVISION)) + .map(Long::parseLong) + .orElse(null); + @Nullable final Long toHistoricalRevision = Optional.ofNullable( + parameters.get(PARAM_TO_HISTORICAL_REVISION)) + .map(Long::parseLong) + .orElse(null); + + @Nullable final Instant fromHistoricalTimestamp = Optional.ofNullable( + parameters.get(PARAM_FROM_HISTORICAL_TIMESTAMP)) + .map(Instant::parse) + .orElse(null); + @Nullable final Instant toHistoricalTimestamp = Optional.ofNullable( + parameters.get(PARAM_TO_HISTORICAL_TIMESTAMP)) + .map(Instant::parse) + .orElse(null); + final CompletionStage facadeStage = signalEnrichmentProvider == null ? CompletableFuture.completedStage(null) : signalEnrichmentProvider.getFacade(ctx.getRequest()); + final var sseSourceStage = facadeStage.thenCompose(facade -> dittoHeadersStage.thenCompose( dittoHeaders -> sseAuthorizationEnforcer.checkAuthorization(ctx, dittoHeaders).thenApply(unused -> { if (filterString != null) { @@ -332,6 +368,37 @@ private Route createSseRoute(final RequestContext ctx, final CompletionStage new IllegalStateException( + "Expected correlation-id in SSE DittoHeaders: " + dittoHeaders)); + final var authorizationContext = dittoHeaders.getAuthorizationContext(); + final Object startStreaming; + if (null != fromHistoricalRevision) { + FeatureToggle + .checkHistoricalApiAccessFeatureEnabled(SubscribeForPersistedEvents.TYPE, dittoHeaders); + startStreaming = SubscribeForPersistedEvents.of(targetThingIds.get(0), + fieldPointer, + fromHistoricalRevision, + null != toHistoricalRevision ? toHistoricalRevision : Long.MAX_VALUE, + dittoHeaders); + } else if (null != fromHistoricalTimestamp) { + FeatureToggle + .checkHistoricalApiAccessFeatureEnabled(SubscribeForPersistedEvents.TYPE, dittoHeaders); + startStreaming = SubscribeForPersistedEvents.of(targetThingIds.get(0), + fieldPointer, + fromHistoricalTimestamp, + toHistoricalTimestamp, + dittoHeaders); + } else { + startStreaming = + StartStreaming.getBuilder(StreamingType.EVENTS, connectionCorrelationId, + authorizationContext) + .withNamespaces(namespaces) + .withFilter(filterString) + .withExtraFields(extraFields) + .build(); + } + final Source publisherSource = SupervisedStream.sourceQueue(10); @@ -339,25 +406,14 @@ private Route createSseRoute(final RequestContext ctx, final CompletionStage { final SupervisedStream.WithQueue withQueue = pair.first(); final KillSwitch killSwitch = pair.second(); - final String connectionCorrelationId = dittoHeaders.getCorrelationId() - .orElseThrow(() -> new IllegalStateException( - "Expected correlation-id in SSE DittoHeaders: " + dittoHeaders)); final var jsonSchemaVersion = dittoHeaders.getSchemaVersion() .orElse(JsonSchemaVersion.LATEST); sseConnectionSupervisor.supervise(withQueue.getSupervisedStream(), connectionCorrelationId, dittoHeaders); - final var authorizationContext = dittoHeaders.getAuthorizationContext(); final var connect = new Connect(withQueue.getSourceQueue(), connectionCorrelationId, STREAMING_TYPE_SSE, jsonSchemaVersion, null, Set.of(), authorizationContext, null); - final var startStreaming = - StartStreaming.getBuilder(StreamingType.EVENTS, connectionCorrelationId, - authorizationContext) - .withNamespaces(namespaces) - .withFilter(filterString) - .withExtraFields(extraFields) - .build(); Patterns.ask(streamingActor, connect, LOCAL_ASK_TIMEOUT) .thenApply(ActorRef.class::cast) .thenAccept(streamingSessionActor -> @@ -369,8 +425,7 @@ private Route createSseRoute(final RequestContext ctx, final CompletionStage - postprocess(jsonifiable, facade, targetThingIds, namespaces, fieldPointer, - fields)) + postprocess(jsonifiable, facade, targetThingIds, namespaces, fieldPointer, fields)) .mapConcat(jsonValues -> jsonValues) .map(jsonValue -> { THINGS_SSE_COUNTER.increment(); @@ -546,8 +601,7 @@ private CompletionStage> postprocess(final SessionedJsonif final Supplier>> emptySupplier = () -> CompletableFuture.completedFuture(Collections.emptyList()); - if (jsonifiable.getJsonifiable() instanceof ThingEvent) { - final ThingEvent event = (ThingEvent) jsonifiable.getJsonifiable(); + if (jsonifiable.getJsonifiable() instanceof ThingEvent event) { final boolean isLiveEvent = StreamingType.isLiveSignal(event); if (!isLiveEvent && namespaceMatches(event, namespaces) && targetThingIdMatches(event, targetThingIds)) { @@ -640,11 +694,21 @@ private static Collection toNonemptyValue(final Thing thing, final Th final JsonObject thingJson = null != fields ? thing.toJson(jsonSchemaVersion, fields) : thing.toJson(jsonSchemaVersion); + @Nullable final JsonValue returnValue; if (!fieldPointer.isEmpty()) { returnValue = thingJson.getValue(fieldPointer).orElse(null); } else { - returnValue = thingJson; + final boolean includeContext = Optional.ofNullable(fields) + .filter(field -> field.getPointers().stream() + .map(JsonPointer::getRoot) + .anyMatch(p -> p.equals(CONTEXT.getPointer().getRoot())) + ).isPresent(); + if (includeContext) { + returnValue = addContext(thingJson.toBuilder(), event).get(fields); + } else { + returnValue = thingJson; + } } return (thingJson.isEmpty() || null == returnValue) ? Collections.emptyList() : Collections.singletonList(returnValue); @@ -694,4 +758,29 @@ private static Counter getCounterFor(final String path) { .tag("path", path); } + /** + * Add a JSON object at {@code _context} key containing e.g. the {@code headers} of the passed + * {@code withDittoHeaders}. + * + * @param objectBuilder the JsonObject build to add the {@code _context} to. + * @param withDittoHeaders the object to extract the {@code DittoHeaders} from. + * @return the built JsonObject including the {@code _context}. + */ + private static JsonObject addContext(final JsonObjectBuilder objectBuilder, + final WithDittoHeaders withDittoHeaders) { + + objectBuilder.set(CONTEXT, JsonObject.newBuilder() + .set("headers", dittoHeadersToJson(withDittoHeaders.getDittoHeaders())) + .build() + ); + return objectBuilder.build(); + } + + private static JsonObject dittoHeadersToJson(final DittoHeaders dittoHeaders) { + return dittoHeaders.entrySet() + .stream() + .map(entry -> JsonFactory.newField(JsonKey.of(entry.getKey()), JsonFactory.newValue(entry.getValue()))) + .collect(JsonCollectors.fieldsToObject()); + } + } diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SessionedJsonifiable.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SessionedJsonifiable.java index 111735f579d..a21ca920ab4 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SessionedJsonifiable.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/SessionedJsonifiable.java @@ -22,6 +22,7 @@ import org.eclipse.ditto.base.model.json.Jsonifiable; import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.gateway.service.streaming.signals.StreamingAck; import org.eclipse.ditto.internal.models.signalenrichment.SignalEnrichmentFacade; import org.eclipse.ditto.internal.utils.pubsub.StreamingType; @@ -141,16 +142,27 @@ static SessionedJsonifiable response(final CommandResponse response) { } /** - * Create a sessioned Jsonifiable for a {@link org.eclipse.ditto.thingsearch.model.signals.events.SubscriptionEvent} + * Create a sessioned Jsonifiable for a {@link SubscriptionEvent} * as response. * - * @param subscriptionEvent the {@link org.eclipse.ditto.thingsearch.model.signals.events.SubscriptionEvent} as response. + * @param subscriptionEvent the {@link SubscriptionEvent} as response. * @return the sessioned Jsonifiable. */ static SessionedJsonifiable subscription(final SubscriptionEvent subscriptionEvent) { return new SessionedResponseErrorOrAck(subscriptionEvent, subscriptionEvent.getDittoHeaders(), null); } + /** + * Create a sessioned Jsonifiable for a {@link StreamingSubscriptionEvent} + * as response. + * + * @param streamingSubscriptionEvent the {@link StreamingSubscriptionEvent} as response. + * @return the sessioned Jsonifiable. + */ + static SessionedJsonifiable streamingSubscription(final StreamingSubscriptionEvent streamingSubscriptionEvent) { + return new SessionedResponseErrorOrAck(streamingSubscriptionEvent, streamingSubscriptionEvent.getDittoHeaders(), null); + } + /** * Create a sessioned Jsonifiable for a stream acknowledgement. * diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingActor.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingActor.java index 09a96f16998..b86130cdf43 100755 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingActor.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingActor.java @@ -17,6 +17,7 @@ import java.util.stream.StreamSupport; import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.edge.service.streaming.StreamingSubscriptionManager; import org.eclipse.ditto.gateway.service.security.authentication.jwt.JwtAuthenticationResultProvider; import org.eclipse.ditto.gateway.service.security.authentication.jwt.JwtValidator; import org.eclipse.ditto.gateway.service.streaming.signals.Connect; @@ -61,6 +62,7 @@ public final class StreamingActor extends AbstractActorWithTimers implements Ret private final JwtValidator jwtValidator; private final JwtAuthenticationResultProvider jwtAuthenticationResultProvider; private final Props subscriptionManagerProps; + private final Props streamingSubscriptionManagerProps; private final DittoDiagnosticLoggingAdapter logger = DittoLoggerFactory.getDiagnosticLoggingAdapter(this); private final HeaderTranslator headerTranslator; private int childCounter = -1; @@ -94,9 +96,13 @@ private StreamingActor(final DittoProtocolSub dittoProtocolSub, this.headerTranslator = headerTranslator; streamingSessionsCounter = DittoMetrics.gauge("streaming_sessions_count"); final ActorSelection commandForwarderSelection = ActorSelection.apply(commandForwarder, ""); + final Materializer materializer = Materializer.createMaterializer(getContext()); subscriptionManagerProps = SubscriptionManager.props(streamingConfig.getSearchIdleTimeout(), pubSubMediator, - commandForwarderSelection, Materializer.createMaterializer(getContext())); + commandForwarderSelection, materializer); + streamingSubscriptionManagerProps = + StreamingSubscriptionManager.props(streamingConfig.getSearchIdleTimeout(), + commandForwarderSelection, materializer); scheduleScrapeStreamSessionsCounter(); } @@ -148,7 +154,8 @@ private Receive createConnectAndMetricsBehavior() { final ActorRef streamingSessionActor = getContext().actorOf( StreamingSessionActor.props(connect, dittoProtocolSub, commandRouter, streamingConfig, headerTranslator, - subscriptionManagerProps, jwtValidator, jwtAuthenticationResultProvider), + subscriptionManagerProps, streamingSubscriptionManagerProps, + jwtValidator, jwtAuthenticationResultProvider), sessionActorName); getSender().tell(streamingSessionActor, ActorRef.noSender()); }) diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActor.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActor.java index 48f5b64d661..cd5597f415b 100755 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActor.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActor.java @@ -29,6 +29,7 @@ import org.eclipse.ditto.base.model.acks.AcknowledgementLabelNotUniqueException; import org.eclipse.ditto.base.model.acks.FatalPubSubException; import org.eclipse.ditto.base.model.auth.AuthorizationContext; +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.entity.id.WithEntityId; import org.eclipse.ditto.base.model.exceptions.DittoHeaderInvalidException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; @@ -41,7 +42,10 @@ import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.edge.service.acknowledgements.AcknowledgementAggregatorActorStarter; import org.eclipse.ditto.edge.service.acknowledgements.AcknowledgementForwarderActor; import org.eclipse.ditto.edge.service.acknowledgements.message.MessageCommandAckRequestSetter; @@ -50,6 +54,7 @@ import org.eclipse.ditto.edge.service.acknowledgements.things.ThingLiveCommandAckRequestSetter; import org.eclipse.ditto.edge.service.acknowledgements.things.ThingModifyCommandAckRequestSetter; import org.eclipse.ditto.edge.service.placeholders.EntityIdPlaceholder; +import org.eclipse.ditto.edge.service.streaming.StreamingSubscriptionManager; import org.eclipse.ditto.gateway.api.GatewayInternalErrorException; import org.eclipse.ditto.gateway.api.GatewayWebsocketSessionAbortedException; import org.eclipse.ditto.gateway.api.GatewayWebsocketSessionClosedException; @@ -92,6 +97,9 @@ import akka.japi.pf.ReceiveBuilder; import akka.pattern.Patterns; import akka.stream.KillSwitch; +import akka.stream.SourceRef; +import akka.stream.javadsl.Keep; +import akka.stream.javadsl.Sink; import akka.stream.javadsl.SourceQueueWithComplete; import scala.PartialFunction; @@ -120,6 +128,7 @@ final class StreamingSessionActor extends AbstractActorWithTimers { private final ActorRef commandForwarder; private final StreamingConfig streamingConfig; private final ActorRef subscriptionManager; + private final ActorRef streamingSubscriptionManager; private final Set outstandingSubscriptionAcks; private final Map streamingSessions; private final JwtValidator jwtValidator; @@ -139,6 +148,7 @@ private StreamingSessionActor(final Connect connect, final StreamingConfig streamingConfig, final HeaderTranslator headerTranslator, final Props subscriptionManagerProps, + final Props streamingSubscriptionManagerProps, final JwtValidator jwtValidator, final JwtAuthenticationResultProvider jwtAuthenticationResultProvider) { @@ -172,6 +182,8 @@ private StreamingSessionActor(final Connect connect, .withCorrelationId(connectionCorrelationId); connect.getSessionExpirationTime().ifPresent(this::startSessionTimeout); subscriptionManager = getContext().actorOf(subscriptionManagerProps, SubscriptionManager.ACTOR_NAME); + streamingSubscriptionManager = getContext().actorOf(streamingSubscriptionManagerProps, + StreamingSubscriptionManager.ACTOR_NAME); declaredAcks = connect.getDeclaredAcknowledgementLabels(); startSubscriptionRefreshTimer(); } @@ -185,6 +197,7 @@ private StreamingSessionActor(final Connect connect, * @param streamingConfig the config to apply for the streaming session. * @param headerTranslator translates headers from external sources or to external sources. * @param subscriptionManagerProps Props of the subscription manager for search protocol. + * @param streamingSubscriptionManagerProps Props of the subscription manager for streaming subscription commands. * @param jwtValidator validator of JWT tokens. * @param jwtAuthenticationResultProvider provider of JWT authentication results. * @return the Akka configuration Props object. @@ -195,6 +208,7 @@ static Props props(final Connect connect, final StreamingConfig streamingConfig, final HeaderTranslator headerTranslator, final Props subscriptionManagerProps, + final Props streamingSubscriptionManagerProps, final JwtValidator jwtValidator, final JwtAuthenticationResultProvider jwtAuthenticationResultProvider) { @@ -205,6 +219,7 @@ static Props props(final Connect connect, streamingConfig, headerTranslator, subscriptionManagerProps, + streamingSubscriptionManagerProps, jwtValidator, jwtAuthenticationResultProvider); } @@ -259,6 +274,7 @@ private Receive createIncomingSignalBehavior() { commandForwarder.forward(liveCommandResponse, getContext())) .match(CommandResponse.class, this::forwardAcknowledgementOrLiveCommandResponse) .match(ThingSearchCommand.class, this::forwardSearchCommand) + .match(StreamingSubscriptionCommand.class, this::forwardStreamingSubscriptionCommand) .match(Signal.class, signal -> // forward signals for which no reply is expected with self return address for downstream errors commandForwarder.tell(signal, getReturnAddress(signal))) @@ -279,6 +295,10 @@ private Receive createOutgoingSignalBehavior() { logger.debug("Got SubscriptionEvent in <{}> session, publishing: {}", type, signal); eventAndResponsePublisher.offer(SessionedJsonifiable.subscription(signal)); }) + .match(StreamingSubscriptionEvent.class, signal -> { + logger.debug("Got StreamingSubscriptionEvent in <{}> session, publishing: {}", type, signal); + eventAndResponsePublisher.offer(SessionedJsonifiable.streamingSubscription(signal)); + }) .match(CommandResponse.class, this::publishResponseOrError) .match(DittoRuntimeException.class, this::publishResponseOrError) .match(Signal.class, this::isSameOrigin, signal -> @@ -312,6 +332,44 @@ private Receive createOutgoingSignalBehavior() { private Receive createPubSubBehavior() { return ReceiveBuilder.create() + .match(SubscribeForPersistedEvents.class, streamPersistedEvents -> { + authorizationContext = streamPersistedEvents.getDittoHeaders().getAuthorizationContext(); + final var session = StreamingSession.of( + streamPersistedEvents.getEntityId() instanceof NamespacedEntityId nsEid ? + List.of(nsEid.getNamespace()) : List.of(), + null, + null, + getSelf(), + logger); + streamingSessions.put(StreamingType.EVENTS, session); + + Patterns.ask(commandForwarder, streamPersistedEvents, streamPersistedEvents.getDittoHeaders() + .getTimeout() + .orElse(Duration.ofSeconds(5)) + ) + .thenApply(response -> (SourceRef) response) + .whenComplete((sourceRef, throwable) -> { + if (null != sourceRef) { + sourceRef.getSource() + .toMat(Sink.actorRef(getSelf(), Control.TERMINATED), Keep.left()) + .run(getContext().getSystem()); + } else if (null != throwable) { + final var dittoRuntimeException = DittoRuntimeException + .asDittoRuntimeException(throwable, + cause -> GatewayInternalErrorException.newBuilder() + .dittoHeaders(DittoHeaders.newBuilder() + .correlationId(connectionCorrelationId) + .build()) + .cause(cause) + .build() + ); + eventAndResponsePublisher.offer(SessionedJsonifiable.error(dittoRuntimeException)); + terminateWebsocketStream(); + } else { + terminateWebsocketStream(); + } + }); + }) .match(StartStreaming.class, startStreaming -> { authorizationContext = startStreaming.getAuthorizationContext(); Criteria criteria; @@ -555,6 +613,11 @@ private void forwardSearchCommand(final ThingSearchCommand searchCommand) { subscriptionManager.tell(searchCommand, getSelf()); } + private void forwardStreamingSubscriptionCommand( + final StreamingSubscriptionCommand streamingSubscriptionCommand) { + streamingSubscriptionManager.tell(streamingSubscriptionCommand, getSelf()); + } + private boolean isSessionAllowedToReceiveSignal(final Signal signal, final StreamingSession session, final StreamingType streamingType) { diff --git a/gateway/service/src/main/resources/gateway.conf b/gateway/service/src/main/resources/gateway.conf index 00cb2f3971d..e056311f1d1 100755 --- a/gateway/service/src/main/resources/gateway.conf +++ b/gateway/service/src/main/resources/gateway.conf @@ -112,7 +112,9 @@ ditto { "live-channel-timeout-strategy", "allow-policy-lockout", "condition", - "live-channel-condition" + "live-channel-condition", + "at-historical-revision", + "at-historical-timestamp" ] } diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalCommandRegistryTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalCommandRegistryTest.java index 4d192f59369..df3a1b3353b 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalCommandRegistryTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalCommandRegistryTest.java @@ -17,6 +17,7 @@ import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoAddConnectionLogEntry; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionIdsByTag; @@ -67,7 +68,8 @@ public GatewayServiceGlobalCommandRegistryTest() { PublishSignal.class, ModifySplitBrainResolver.class, CleanupPersistence.class, - SudoAddConnectionLogEntry.class + SudoAddConnectionLogEntry.class, + SubscribeForPersistedEvents.class ); } diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalEventRegistryTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalEventRegistryTest.java index e7b03a4ae76..d658562e4aa 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalEventRegistryTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/starter/GatewayServiceGlobalEventRegistryTest.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.gateway.service.starter; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases; import org.eclipse.ditto.policies.model.signals.events.ResourceDeleted; @@ -29,7 +30,8 @@ public GatewayServiceGlobalEventRegistryTest() { FeatureDeleted.class, ThingsOutOfSync.class, SubscriptionCreated.class, - ThingSnapshotTaken.class + ThingSnapshotTaken.class, + StreamingSubscriptionComplete.class ); } diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorHeaderInteractionTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorHeaderInteractionTest.java index d891436f2c8..b3f11decf00 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorHeaderInteractionTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorHeaderInteractionTest.java @@ -116,6 +116,7 @@ public static Collection getParameters() { private final TestProbe eventResponsePublisherProbe = TestProbe.apply("eventAndResponsePublisher", actorSystem); private final TestProbe commandRouterProbe = TestProbe.apply("commandRouter", actorSystem); private final TestProbe subscriptionManagerProbe = TestProbe.apply("subscriptionManager", actorSystem); + private final TestProbe streamingSubscriptionManagerProbe = TestProbe.apply("streamingSubscriptionManager", actorSystem); private final DittoProtocolSub dittoProtocolSub = Mockito.mock(DittoProtocolSub.class); private final SourceQueueWithComplete sourceQueue; @@ -198,7 +199,9 @@ private ActorRef createStreamingSessionActor() { null); final Props props = StreamingSessionActor.props(connect, dittoProtocolSub, commandRouterProbe.ref(), DefaultStreamingConfig.of(ConfigFactory.empty()), HeaderTranslator.empty(), - Props.create(TestProbeForwarder.class, subscriptionManagerProbe), Mockito.mock(JwtValidator.class), + Props.create(TestProbeForwarder.class, subscriptionManagerProbe), + Props.create(TestProbeForwarder.class, streamingSubscriptionManagerProbe), + Mockito.mock(JwtValidator.class), Mockito.mock(JwtAuthenticationResultProvider.class)); final ActorRef createdActor = actorSystem.actorOf(props); createdActors.add(createdActor); diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorTest.java index 4ae7337b51f..fd498c2dc4a 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/streaming/actors/StreamingSessionActorTest.java @@ -397,6 +397,7 @@ private Props getProps(final String... declaredAcks) { DefaultStreamingConfig.of(ConfigFactory.empty()), HeaderTranslator.empty(), Props.create(Actor.class, () -> new TestActor(new LinkedBlockingDeque<>())), + Props.create(Actor.class, () -> new TestActor(new LinkedBlockingDeque<>())), mockValidator, mockAuthenticationResultProvider); } diff --git a/internal/utils/config/src/main/resources/ditto-devops.conf b/internal/utils/config/src/main/resources/ditto-devops.conf index 7679e1cb28d..d0fd0e1bac5 100644 --- a/internal/utils/config/src/main/resources/ditto-devops.conf +++ b/internal/utils/config/src/main/resources/ditto-devops.conf @@ -16,5 +16,9 @@ ditto.devops { // enables/disables the WoT (Web of Things) integration feature wot-integration-enabled = true wot-integration-enabled = ${?DITTO_DEVOPS_FEATURE_WOT_INTEGRATION_ENABLED} + + // enables/disables the historical API access feature + historical-apis-enabled = true + historical-apis-enabled = ${?DITTO_DEVOPS_FEATURE_HISTORICAL_APIS_ENABLED} } } diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/AbstractMongoEventAdapter.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/AbstractMongoEventAdapter.java index 75ea8d043c6..41e34a8462b 100644 --- a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/AbstractMongoEventAdapter.java +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/AbstractMongoEventAdapter.java @@ -12,20 +12,23 @@ */ package org.eclipse.ditto.internal.utils.persistence.mongo; +import java.util.Optional; import java.util.Set; -import javax.annotation.Nullable; - import org.bson.BsonDocument; import org.bson.BsonValue; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.DittoHeadersBuilder; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.events.Event; import org.eclipse.ditto.base.model.signals.events.EventRegistry; import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; +import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.json.JsonParseException; import org.eclipse.ditto.json.JsonValue; import org.slf4j.Logger; @@ -43,13 +46,21 @@ public abstract class AbstractMongoEventAdapter> implements E private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMongoEventAdapter.class); - @Nullable protected final ExtendedActorSystem system; + /** + * Internal header for persisting the historical headers for events. + */ + public static final JsonFieldDefinition HISTORICAL_EVENT_HEADERS = JsonFieldDefinition.ofJsonObject( + "__hh"); + + protected final ExtendedActorSystem system; protected final EventRegistry eventRegistry; + private final EventConfig eventConfig; - protected AbstractMongoEventAdapter(@Nullable final ExtendedActorSystem system, - final EventRegistry eventRegistry) { + protected AbstractMongoEventAdapter(final ExtendedActorSystem system, + final EventRegistry eventRegistry, final EventConfig eventConfig) { this.system = system; this.eventRegistry = eventRegistry; + this.eventConfig = eventConfig; } @Override @@ -66,9 +77,9 @@ public String manifest(final Object event) { public Object toJournal(final Object event) { if (event instanceof Event theEvent) { final JsonSchemaVersion schemaVersion = theEvent.getImplementedSchemaVersion(); - final JsonObject jsonObject = performToJournalMigration( + final JsonObject jsonObject = performToJournalMigration(theEvent, theEvent.toJson(schemaVersion, FieldType.regularOrSpecial()) - ); + ).build(); final BsonDocument bson = DittoBsonJson.getInstance().parse(jsonObject); final Set tags = theEvent.getDittoHeaders().getJournalTags(); return new Tagged(bson, tags); @@ -84,8 +95,12 @@ public EventSeq fromJournal(final Object event, final String manifest) { try { final JsonObject jsonObject = jsonValue.asObject() .setValue(EventsourcedEvent.JsonFields.REVISION.getPointer(), Event.DEFAULT_REVISION); + final DittoHeaders dittoHeaders = jsonObject.getValue(HISTORICAL_EVENT_HEADERS) + .map(obj -> DittoHeaders.newBuilder(obj).build()) + .orElse(DittoHeaders.empty()); + final T result = - eventRegistry.parse(performFromJournalMigration(jsonObject), DittoHeaders.empty()); + eventRegistry.parse(performFromJournalMigration(jsonObject), dittoHeaders); return EventSeq.single(result); } catch (final JsonParseException | DittoRuntimeException e) { if (system != null) { @@ -105,11 +120,22 @@ public EventSeq fromJournal(final Object event, final String manifest) { * Performs an optional migration of the passed in {@code jsonObject} (which is the JSON representation of the * {@link Event} to persist) just before it is transformed to Mongo BSON and inserted into the "journal" collection. * - * @param jsonObject the JsonObject representation of the {@link Event} to persist. + * @param event the event to apply journal migration for. + * @param jsonObject the JsonObject representation of the {@link org.eclipse.ditto.base.model.signals.events.Event} to persist. * @return the adjusted/migrated JsonObject to store. */ - protected JsonObject performToJournalMigration(final JsonObject jsonObject) { - return jsonObject; + protected JsonObjectBuilder performToJournalMigration(final Event event, final JsonObject jsonObject) { + return jsonObject.toBuilder() + .set(HISTORICAL_EVENT_HEADERS, calculateHistoricalHeaders(event.getDittoHeaders()).toJson()); + } + + private DittoHeaders calculateHistoricalHeaders(final DittoHeaders dittoHeaders) { + final DittoHeadersBuilder historicalHeadersBuilder = DittoHeaders.newBuilder(); + eventConfig.getHistoricalHeadersToPersist().forEach(headerKeyToPersist -> + Optional.ofNullable(dittoHeaders.get(headerKeyToPersist)) + .ifPresent(value -> historicalHeadersBuilder.putHeader(headerKeyToPersist, value)) + ); + return historicalHeadersBuilder.build(); } /** diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfig.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfig.java new file mode 100644 index 00000000000..0f053e50b96 --- /dev/null +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfig.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.internal.utils.persistence.mongo.config; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.internal.utils.config.ConfigWithFallback; +import org.eclipse.ditto.internal.utils.config.ScopedConfig; + +import com.typesafe.config.Config; + +/** + * This class implements the config for the handling of event journal entries. + */ +@Immutable +public final class DefaultEventConfig implements EventConfig { + + private static final String CONFIG_PATH = "event"; + + private final List historicalHeadersToPersist; + + private DefaultEventConfig(final ScopedConfig config) { + historicalHeadersToPersist = Collections.unmodifiableList(new ArrayList<>( + config.getStringList(EventConfigValue.HISTORICAL_HEADERS_TO_PERSIST.getConfigPath()) + )); + } + + /** + * Returns an instance of the default event journal config based on the settings of the specified Config. + * + * @param config is supposed to provide the settings of the event journal config at {@value #CONFIG_PATH}. + * @return instance + * @throws org.eclipse.ditto.internal.utils.config.DittoConfigError if {@code config} is invalid. + */ + public static DefaultEventConfig of(final Config config) { + return new DefaultEventConfig( + ConfigWithFallback.newInstance(config, CONFIG_PATH, EventConfigValue.values())); + } + + @Override + public List getHistoricalHeadersToPersist() { + return historicalHeadersToPersist; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DefaultEventConfig that = (DefaultEventConfig) o; + return Objects.equals(historicalHeadersToPersist, that.historicalHeadersToPersist); + } + + @Override + public int hashCode() { + return Objects.hash(historicalHeadersToPersist); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + + "historicalHeadersToPersist=" + historicalHeadersToPersist + + "]"; + } + +} diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultSnapshotConfig.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultSnapshotConfig.java index 40c25fd03af..e85967e7d93 100644 --- a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultSnapshotConfig.java +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultSnapshotConfig.java @@ -23,7 +23,7 @@ import com.typesafe.config.Config; /** - * This class implements the config for the handling of snapshots of policy entities. + * This class implements the config for the handling of snapshots of entities. */ @Immutable public final class DefaultSnapshotConfig implements SnapshotConfig { diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/EventConfig.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/EventConfig.java new file mode 100644 index 00000000000..436fdcc2286 --- /dev/null +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/EventConfig.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.internal.utils.persistence.mongo.config; + +import java.util.List; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; +import org.eclipse.ditto.internal.utils.config.KnownConfigValue; + +/** + * Provides configuration settings for the handling entity journal events. + */ +@Immutable +public interface EventConfig { + + /** + * Returns the DittoHeader keys to additionally persist for events in the event journal, e.g. in order + * to enable additional context/information for an audit log. + * + * @return the historical headers to persist into the event journal. + */ + List getHistoricalHeadersToPersist(); + + + /** + * An enumeration of the known config path expressions and their associated default values for + * {@code SnapshotConfig}. + */ + enum EventConfigValue implements KnownConfigValue { + + /** + * The DittoHeaders to persist when persisting events to the journal. + */ + HISTORICAL_HEADERS_TO_PERSIST("historical-headers-to-persist", List.of( + DittoHeaderDefinition.ORIGINATOR.getKey(), + DittoHeaderDefinition.CORRELATION_ID.getKey() + )); + + private final String path; + private final Object defaultValue; + + EventConfigValue(final String thePath, final Object theDefaultValue) { + path = thePath; + defaultValue = theDefaultValue; + } + + @Override + public Object getDefaultValue() { + return defaultValue; + } + + @Override + public String getConfigPath() { + return path; + } + + } + +} diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/MongoReadJournal.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/MongoReadJournal.java index 2e122addce1..d163d83ceab 100644 --- a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/MongoReadJournal.java +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/MongoReadJournal.java @@ -13,6 +13,7 @@ package org.eclipse.ditto.internal.utils.persistence.mongo.streaming; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -55,9 +56,19 @@ import akka.Done; import akka.NotUsed; import akka.actor.ActorSystem; +import akka.contrib.persistence.mongodb.JavaDslMongoReadJournal; import akka.contrib.persistence.mongodb.JournallingFieldNames$; import akka.contrib.persistence.mongodb.SnapshottingFieldNames$; import akka.japi.Pair; +import akka.persistence.query.EventEnvelope; +import akka.persistence.query.Offset; +import akka.persistence.query.PersistenceQuery; +import akka.persistence.query.javadsl.CurrentEventsByPersistenceIdQuery; +import akka.persistence.query.javadsl.CurrentEventsByTagQuery; +import akka.persistence.query.javadsl.CurrentPersistenceIdsQuery; +import akka.persistence.query.javadsl.EventsByPersistenceIdQuery; +import akka.persistence.query.javadsl.EventsByTagQuery; +import akka.persistence.query.javadsl.PersistenceIdsQuery; import akka.stream.Attributes; import akka.stream.Materializer; import akka.stream.RestartSettings; @@ -67,7 +78,7 @@ import akka.stream.javadsl.Source; /** - * Reads the event journal of com.github.scullxbones.akka-persistence-mongo plugin. + * Reads the event journal of {@code com.github.scullxbones.akka-persistence-mongo} plugin. * In the Akka system configuration, *
    *
  • @@ -81,12 +92,18 @@ *
*/ @AllValuesAreNonnullByDefault -public final class MongoReadJournal { +public final class MongoReadJournal implements CurrentEventsByPersistenceIdQuery, + CurrentEventsByTagQuery, CurrentPersistenceIdsQuery, EventsByPersistenceIdQuery, EventsByTagQuery, + PersistenceIdsQuery { /** - * ID field of documents delivered by the read journal. + * ID field of documents delivered by the journal collection. + */ + public static final String J_ID = JournallingFieldNames$.MODULE$.ID(); + + /** + * ID field of documents delivered by the snaps collection. */ - private static final String J_ID = JournallingFieldNames$.MODULE$.ID(); public static final String S_ID = J_ID; /** @@ -125,6 +142,11 @@ public final class MongoReadJournal { */ public static final String S_SN = SnapshottingFieldNames$.MODULE$.SEQUENCE_NUMBER(); + /** + * Document field of the timestamp of snapshots. + */ + public static final String S_TS = SnapshottingFieldNames$.MODULE$.TIMESTAMP(); + private static final String S_SERIALIZED_SNAPSHOT = "s2"; /** @@ -147,8 +169,11 @@ public final class MongoReadJournal { private final DittoMongoClient mongoClient; private final IndexInitializer indexInitializer; + private final JavaDslMongoReadJournal akkaReadJournal; + private MongoReadJournal(final String journalCollection, final String snapsCollection, + final String readJournalConfigurationKey, final DittoMongoClient mongoClient, final ActorSystem actorSystem) { @@ -157,6 +182,8 @@ private MongoReadJournal(final String journalCollection, this.mongoClient = mongoClient; final var materializer = SystemMaterializer.get(actorSystem).materializer(); indexInitializer = IndexInitializer.of(mongoClient.getDefaultDatabase(), materializer); + akkaReadJournal = PersistenceQuery.get(actorSystem) + .getReadJournalFor(JavaDslMongoReadJournal.class, readJournalConfigurationKey); } /** @@ -188,7 +215,12 @@ public static MongoReadJournal newInstance(final Config config, final DittoMongo getOverrideCollectionName(config.getConfig(autoStartJournalKey), JOURNAL_COLLECTION_NAME_KEY); final String snapshotCollection = getOverrideCollectionName(config.getConfig(autoStartSnapsKey), SNAPS_COLLECTION_NAME_KEY); - return new MongoReadJournal(journalCollection, snapshotCollection, mongoClient, actorSystem); + return new MongoReadJournal(journalCollection, + snapshotCollection, + autoStartJournalKey + "-read", + mongoClient, + actorSystem + ); } /** @@ -397,6 +429,29 @@ public Source getJournalPidsAboveWithTag(final String lowerBoun .mapConcat(pids -> pids); } + /** + * A Source retrieving a single revision/sequence number of type {@code long} for the last snapshot sequence number + * available for the passed {@code pid} and before the passed {@code timestamp}. + * + * @param pid the persistenceId to find out the last snapshot sequence number for. + * @param timestamp the timestamp to use as selection criteria for the snapshot sequence number to find out. + * @return a Source of a single element with the determined snapshot sequence number. + */ + public Source getLastSnapshotSequenceNumberBeforeTimestamp(final String pid, + final Instant timestamp) { + + final Bson filter = Filters.and( + Filters.eq(S_PROCESSOR_ID, pid), + Filters.lte(S_TS, timestamp.toEpochMilli()) + ); + return getSnapshotStore().flatMapConcat(snaps -> Source.fromPublisher(snaps + .find(filter) + .projection(Projections.include(S_SN)) + .sort(Sorts.descending(S_SN)) + .first() + )).map(document -> document.getLong(S_SN)); + } + /** * Retrieve all latest snapshots with unique PIDs in snapshot store above a lower bound. * Does not limit database access in any way. @@ -412,7 +467,7 @@ public Source getNewestSnapshotsAbove(final String lowerBound final Materializer mat, final String... snapshotFields) { - return getNewestSnapshotsAbove(lowerBoundPid, batchSize, false, mat, snapshotFields); + return getNewestSnapshotsAbove(lowerBoundPid, batchSize, false, Duration.ZERO, mat, snapshotFields); } /** @@ -422,6 +477,8 @@ public Source getNewestSnapshotsAbove(final String lowerBound * @param lowerBoundPid the lower-bound PID. * @param batchSize how many snapshots to read in 1 query. * @param includeDeleted whether to include deleted snapshots. + * @param minAgeFromNow the minimum age (based on {@code Instant.now()}) the snapshot must have in order to get + * selected. * @param mat the materializer. * @param snapshotFields snapshot fields to project out. * @return source of newest snapshots with unique PIDs. @@ -429,14 +486,20 @@ public Source getNewestSnapshotsAbove(final String lowerBound public Source getNewestSnapshotsAbove(final String lowerBoundPid, final int batchSize, final boolean includeDeleted, + final Duration minAgeFromNow, final Materializer mat, final String... snapshotFields) { return getSnapshotStore() .withAttributes(Attributes.inputBuffer(1, 1)) .flatMapConcat(snapshotStore -> - listNewestSnapshots(snapshotStore, SnapshotFilter.of(lowerBoundPid), batchSize, includeDeleted, mat, - snapshotFields) + listNewestSnapshots(snapshotStore, + SnapshotFilter.of(lowerBoundPid, minAgeFromNow), + batchSize, + includeDeleted, + mat, + snapshotFields + ) ) .mapConcat(pids -> pids); } @@ -460,8 +523,7 @@ public Source getNewestSnapshotsAbove( return getSnapshotStore() .withAttributes(Attributes.inputBuffer(1, 1)) .flatMapConcat(snapshotStore -> - listNewestSnapshots(snapshotStore, snapshotFilter, batchSize, false, mat, - snapshotFields) + listNewestSnapshots(snapshotStore, snapshotFilter, batchSize, false, mat, snapshotFields) ) .mapConcat(pids -> pids); } @@ -509,12 +571,12 @@ public Source, NotUsed> getSmallestSnapshotSeqNo(final String pid * @return source of the delete result. */ public Source deleteEvents(final String pid, final long minSeqNr, final long maxSeqNr) { + + final Bson filter = Filters.and(Filters.eq(J_PROCESSOR_ID, pid), + Filters.gte(J_TO, minSeqNr), + Filters.lte(J_TO, maxSeqNr)); return getJournal() - .flatMapConcat(journal -> Source.fromPublisher( - journal.deleteMany(Filters.and(Filters.eq(J_PROCESSOR_ID, pid), - Filters.gte(J_TO, minSeqNr), - Filters.lte(J_TO, maxSeqNr))) - )); + .flatMapConcat(journal -> Source.fromPublisher(journal.deleteMany(filter))); } /** @@ -526,12 +588,46 @@ public Source deleteEvents(final String pid, final long m * @return source of the delete result. */ public Source deleteSnapshots(final String pid, final long minSeqNr, final long maxSeqNr) { + + final Bson filter = Filters.and(Filters.eq(S_PROCESSOR_ID, pid), + Filters.gte(S_SN, minSeqNr), + Filters.lte(S_SN, maxSeqNr)); return getSnapshotStore() - .flatMapConcat(snaps -> Source.fromPublisher( - snaps.deleteMany(Filters.and(Filters.eq(S_PROCESSOR_ID, pid), - Filters.gte(S_SN, minSeqNr), - Filters.lte(S_SN, maxSeqNr))) - )); + .flatMapConcat(snaps -> Source.fromPublisher(snaps.deleteMany(filter))); + } + + + @Override + public Source currentEventsByPersistenceId(final String persistenceId, + final long fromSequenceNr, + final long toSequenceNr) { + return akkaReadJournal.currentEventsByPersistenceId(persistenceId, fromSequenceNr, toSequenceNr); + } + + @Override + public Source currentEventsByTag(final String tag, final Offset offset) { + return akkaReadJournal.currentEventsByTag(tag, offset); + } + + @Override + public Source currentPersistenceIds() { + return akkaReadJournal.currentPersistenceIds(); + } + + @Override + public Source eventsByPersistenceId(final String persistenceId, final long fromSequenceNr, + final long toSequenceNr) { + return akkaReadJournal.eventsByPersistenceId(persistenceId, fromSequenceNr, toSequenceNr); + } + + @Override + public Source eventsByTag(final String tag, final Offset offset) { + return akkaReadJournal.eventsByTag(tag, offset); + } + + @Override + public Source persistenceIds() { + return akkaReadJournal.persistenceIds(); } private Source, NotUsed> listPidsInJournal(final MongoCollection journal, @@ -570,11 +666,16 @@ private Source, NotUsed> listNewestSnapshots(final MongoCollectio final Materializer mat, final String... snapshotFields) { - return unfoldBatchedSource(filter.getLowerBoundPid(), + return unfoldBatchedSource(filter.lowerBoundPid(), mat, SnapshotBatch::maxPid, - actualStartPid -> listNewestActiveSnapshotsByBatch(snapshotStore, filter.withLowerBound(actualStartPid), batchSize, - includeDeleted, snapshotFields)) + actualStartPid -> listNewestActiveSnapshotsByBatch(snapshotStore, + filter.withLowerBound(actualStartPid), + batchSize, + includeDeleted, + snapshotFields + ) + ) .mapConcat(x -> x) .map(SnapshotBatch::items); } @@ -729,8 +830,8 @@ private static Source listNewestActiveSnapshotsByBatch( final String... snapshotFields) { final List pipeline = new ArrayList<>(5); - // optional match stage - snapshotFilter.toMongoFilter().ifPresent(bson -> pipeline.add(Aggregates.match(bson))); + // match stage + pipeline.add(Aggregates.match(snapshotFilter.toMongoFilter())); // sort stage pipeline.add(Aggregates.sort(Sorts.orderBy(Sorts.ascending(S_PROCESSOR_ID), Sorts.descending(S_SN)))); @@ -778,7 +879,9 @@ private static Source listNewestActiveSnapshotsByBatch( if (theMaxPid == null) { return Source.empty(); } else { - return Source.single(new SnapshotBatch(theMaxPid, document.getList(items, Document.class))); + final SnapshotBatch snapshotBatch = + new SnapshotBatch(theMaxPid, document.getList(items, Document.class)); + return Source.single(snapshotBatch); } }); } diff --git a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/SnapshotFilter.java b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/SnapshotFilter.java index b5519c0c157..d1cc891adb7 100644 --- a/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/SnapshotFilter.java +++ b/internal/utils/persistence/src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/streaming/SnapshotFilter.java @@ -13,9 +13,12 @@ package org.eclipse.ditto.internal.utils.persistence.mongo.streaming; -import java.util.Optional; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; import org.bson.conversions.Bson; +import org.bson.types.ObjectId; import com.mongodb.client.model.Filters; @@ -23,16 +26,27 @@ /** * A record that hold optional filters for retrieving snapshots from persistence. + * + * @param lowerBoundPid the lower-bound pid from which to start reading the snapshots + * @param pidFilter the regex applied to the pid to filter the snapshots + * @param minAgeFromNow the minimum age (based on {@code Instant.now()}) the snapshot must have in order to get + * selected */ -public record SnapshotFilter(String lowerBoundPid, String pidFilter) { +public record SnapshotFilter(String lowerBoundPid, String pidFilter, Duration minAgeFromNow) { /** * Document field of PID in snapshot stores. */ private static final String S_PROCESSOR_ID = SnapshottingFieldNames$.MODULE$.PROCESSOR_ID(); - static SnapshotFilter of(final String lowerBoundPid) { - return new SnapshotFilter(lowerBoundPid, ""); + /** + * @param lowerBoundPid the lower-bound pid from which to start reading the snapshots + * @param minAgeFromNow the minimum age (based on {@code Instant.now()}) the snapshot must have in order to get + * selected. + * @return new instance of SnapshotFilter + */ + static SnapshotFilter of(final String lowerBoundPid, final Duration minAgeFromNow) { + return of(lowerBoundPid, "", minAgeFromNow); } /** @@ -41,14 +55,19 @@ static SnapshotFilter of(final String lowerBoundPid) { * @return new instance of SnapshotFilter */ public static SnapshotFilter of(final String lowerBoundPid, final String pidFilter) { - return new SnapshotFilter(lowerBoundPid, pidFilter); + return of(lowerBoundPid, pidFilter, Duration.ZERO); } /** - * @return the lower-bound pid + * @param lowerBoundPid the lower-bound pid from which to start reading the snapshots + * @param pidFilter the regex applied to the pid to filter the snapshots + * @param minAgeFromNow the minimum age (based on {@code Instant.now()}) the snapshot must have in order to get + * selected. + * @return new instance of SnapshotFilter */ - String getLowerBoundPid() { - return lowerBoundPid; + public static SnapshotFilter of(final String lowerBoundPid, final String pidFilter, + final Duration minAgeFromNow) { + return new SnapshotFilter(lowerBoundPid, pidFilter, minAgeFromNow); } /** @@ -56,21 +75,32 @@ String getLowerBoundPid() { * @return a new instance of SnapshotFilter with the new lower-bound pid set */ SnapshotFilter withLowerBound(final String newLowerBoundPid) { - return new SnapshotFilter(newLowerBoundPid, pidFilter); + return new SnapshotFilter(newLowerBoundPid, pidFilter, minAgeFromNow); } /** * @return a Bson filter that can be used in a mongo query to filter the snapshots or an empty Optional if no filter was set */ - Optional toMongoFilter() { + Bson toMongoFilter() { + final Bson filter; if (!lowerBoundPid.isEmpty() && !pidFilter.isEmpty()) { - return Optional.of(Filters.and(getLowerBoundFilter(), getNamespacesFilter())); + filter = Filters.and(getLowerBoundFilter(), getNamespacesFilter()); } else if (!lowerBoundPid.isEmpty()) { - return Optional.of(getLowerBoundFilter()); + filter = getLowerBoundFilter(); } else if (!pidFilter.isEmpty()) { - return Optional.of(getNamespacesFilter()); + filter = getNamespacesFilter(); + } else { + filter = Filters.empty(); + } + + if (minAgeFromNow.isZero()) { + return filter; } else { - return Optional.empty(); + final Date nowMinusMinAgeFromNow = Date.from(Instant.now().minus(minAgeFromNow)); + final Bson eventRetentionFilter = Filters.lt("_id", + ObjectId.getSmallestWithDate(nowMinusMinAgeFromNow) + ); + return Filters.and(filter, eventRetentionFilter); } } diff --git a/internal/utils/persistence/src/test/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfigTest.java b/internal/utils/persistence/src/test/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfigTest.java new file mode 100644 index 00000000000..97618879b77 --- /dev/null +++ b/internal/utils/persistence/src/test/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/DefaultEventConfigTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.internal.utils.persistence.mongo.config; + +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.List; + +import org.assertj.core.api.JUnitSoftAssertions; +import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import nl.jqno.equalsverifier.EqualsVerifier; + +/** + * Unit test for {@link DefaultEventConfig}. + */ +public final class DefaultEventConfigTest { + + private static Config snapshotTestConf; + + @Rule + public final JUnitSoftAssertions softly = new JUnitSoftAssertions(); + + @BeforeClass + public static void initTestFixture() { + snapshotTestConf = ConfigFactory.load("event-test"); + } + + @Test + public void assertImmutability() { + assertInstancesOf(DefaultEventConfig.class, areImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(DefaultEventConfig.class) + .usingGetClass() + .verify(); + } + + @Test + public void underTestReturnsDefaultValuesIfBaseConfigWasEmpty() { + final DefaultEventConfig underTest = DefaultEventConfig.of(ConfigFactory.empty()); + + softly.assertThat(underTest.getHistoricalHeadersToPersist()) + .as(EventConfig.EventConfigValue.HISTORICAL_HEADERS_TO_PERSIST.getConfigPath()) + .isEqualTo(EventConfig.EventConfigValue.HISTORICAL_HEADERS_TO_PERSIST.getDefaultValue()); + } + + @Test + public void underTestReturnsValuesOfConfigFile() { + final DefaultEventConfig underTest = DefaultEventConfig.of(snapshotTestConf); + + softly.assertThat(underTest.getHistoricalHeadersToPersist()) + .as(EventConfig.EventConfigValue.HISTORICAL_HEADERS_TO_PERSIST.getConfigPath()) + .isEqualTo(List.of(DittoHeaderDefinition.ORIGINATOR.getKey(), "foo")); + } +} diff --git a/internal/utils/persistence/src/test/resources/event-test.conf b/internal/utils/persistence/src/test/resources/event-test.conf new file mode 100644 index 00000000000..3b84b50ef7a --- /dev/null +++ b/internal/utils/persistence/src/test/resources/event-test.conf @@ -0,0 +1,6 @@ +event { + historical-headers-to-persist = [ + "ditto-originator" + "foo" + ] +} diff --git a/internal/utils/persistence/src/test/resources/mongo-read-journal-test.conf b/internal/utils/persistence/src/test/resources/mongo-read-journal-test.conf index c1a2b8663ca..1905bbf9f7e 100644 --- a/internal/utils/persistence/src/test/resources/mongo-read-journal-test.conf +++ b/internal/utils/persistence/src/test/resources/mongo-read-journal-test.conf @@ -32,6 +32,17 @@ akka-contrib-mongodb-persistence-test-journal { } } +akka-contrib-mongodb-persistence-test-journal-read { + class = "akka.contrib.persistence.mongodb.MongoReadJournal" + + overrides { + journal-collection = "test_journal" + journal-index = "test_journal_index" + realtime-collection = "test_realtime" + metadata-collection = "test_metadata" + } +} + akka-contrib-mongodb-persistence-test-snapshots { class = "akka.contrib.persistence.mongodb.MongoSnapshots" diff --git a/internal/utils/persistence/src/test/resources/test.conf b/internal/utils/persistence/src/test/resources/test.conf index a975197c7fa..cd714bab1ac 100644 --- a/internal/utils/persistence/src/test/resources/test.conf +++ b/internal/utils/persistence/src/test/resources/test.conf @@ -132,6 +132,17 @@ akka-contrib-mongodb-persistence-test-journal { } } +akka-contrib-mongodb-persistence-test-journal-read { + class = "akka.contrib.persistence.mongodb.MongoReadJournal" + + overrides { + journal-collection = "test_journal" + journal-index = "test_journal_index" + realtime-collection = "test_realtime" + metadata-collection = "test_metadata" + } +} + akka-contrib-mongodb-persistence-test-snapshots { class = "akka.persistence.inmemory.snapshot.InMemorySnapshotStore" diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceActor.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceActor.java index 5e0746b82cd..27e9d9178de 100755 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceActor.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceActor.java @@ -13,30 +13,40 @@ package org.eclipse.ditto.internal.utils.persistentactors; import java.time.Duration; +import java.time.Instant; +import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.Consumer; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; +import org.bson.BsonDocument; import org.eclipse.ditto.base.api.commands.sudo.SudoCommand; import org.eclipse.ditto.base.model.entity.id.EntityId; import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.DittoHeadersSettable; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.json.Jsonifiable; +import org.eclipse.ditto.base.model.signals.FeatureToggle; import org.eclipse.ditto.base.model.signals.commands.Command; -import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; +import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry; import org.eclipse.ditto.internal.utils.akka.PingCommand; import org.eclipse.ditto.internal.utils.akka.PingCommandResponse; import org.eclipse.ditto.internal.utils.config.ScopedConfig; import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces; import org.eclipse.ditto.internal.utils.persistence.SnapshotAdapter; +import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; +import org.eclipse.ditto.internal.utils.persistence.mongo.DittoBsonJson; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.commands.CommandStrategy; import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.internal.utils.persistentactors.results.Result; @@ -46,15 +56,21 @@ import org.eclipse.ditto.internal.utils.tracing.span.SpanOperationName; import org.eclipse.ditto.internal.utils.tracing.span.SpanTagKey; import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonValue; import akka.actor.ActorRef; +import akka.actor.Cancellable; import akka.japi.pf.ReceiveBuilder; import akka.persistence.RecoveryCompleted; import akka.persistence.RecoveryTimedOut; import akka.persistence.SaveSnapshotFailure; import akka.persistence.SaveSnapshotSuccess; import akka.persistence.SnapshotOffer; +import akka.persistence.SnapshotProtocol; +import akka.persistence.SnapshotSelectionCriteria; +import akka.persistence.query.EventEnvelope; +import akka.stream.javadsl.Sink; import scala.Option; /** @@ -72,7 +88,8 @@ public abstract class AbstractPersistenceActor< S extends Jsonifiable.WithFieldSelectorAndPredicate, I extends EntityId, K, - E extends Event> extends AbstractPersistentActorWithTimersAndCleanup implements ResultVisitor { + E extends EventsourcedEvent> + extends AbstractPersistentActorWithTimersAndCleanup implements ResultVisitor { /** * An event journal {@code Tag} used to tag journal entries managed by a PersistenceActor as "always alive" meaning @@ -83,6 +100,7 @@ public abstract class AbstractPersistenceActor< private final SnapshotAdapter snapshotAdapter; private final Receive handleEvents; private final Receive handleCleanups; + private final MongoReadJournal mongoReadJournal; private long lastSnapshotRevision; private long confirmedSnapshotRevision; @@ -104,10 +122,12 @@ public abstract class AbstractPersistenceActor< * Instantiate the actor. * * @param entityId the entity ID. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the entity. */ @SuppressWarnings("unchecked") - protected AbstractPersistenceActor(final I entityId) { + protected AbstractPersistenceActor(final I entityId, final MongoReadJournal mongoReadJournal) { this.entityId = entityId; + this.mongoReadJournal = mongoReadJournal; final var actorSystem = context().system(); final var dittoExtensionsConfig = ScopedConfig.dittoExtension(actorSystem.settings().config()); this.snapshotAdapter = SnapshotAdapter.get(actorSystem, dittoExtensionsConfig); @@ -190,6 +210,20 @@ protected void onEntityModified() { */ protected abstract DittoRuntimeExceptionBuilder newNotAccessibleExceptionBuilder(); + /** + * @param revision the revision which could not be resolved in the entity history. + * @return An exception builder to respond to unexpected commands addressed to a nonexistent historical entity at a + * given {@code revision}. + */ + protected abstract DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(long revision); + + /** + * @param timestamp the timestamp which could not be resolved in the entity history. + * @return An exception builder to respond to unexpected commands addressed to a nonexistent historical entity at a + * given {@code timestamp}. + */ + protected abstract DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(Instant timestamp); + /** * Publish an event. * @@ -292,6 +326,8 @@ protected void becomeCreatedHandler() { final CommandStrategy commandStrategy = getCreatedStrategy(); final Receive receive = handleCleanups.orElse(ReceiveBuilder.create() + .match(commandStrategy.getMatchingClass(), this::isHistoricalRetrieveCommand, + this::handleHistoricalRetrieveCommand) .match(commandStrategy.getMatchingClass(), commandStrategy::isDefined, this::handleByCommandStrategy) .match(PersistEmptyEvent.class, this::handlePersistEmptyEvent) .match(CheckForActivity.class, this::checkForActivity) @@ -308,6 +344,145 @@ protected void becomeCreatedHandler() { scheduleSnapshot(); } + private boolean isHistoricalRetrieveCommand(final C command) { + final DittoHeaders headers = command.getDittoHeaders(); + return command.getCategory().equals(Command.Category.QUERY) && ( + headers.containsKey(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey()) || + headers.containsKey(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey()) + ); + } + + private void handleHistoricalRetrieveCommand(final C command) { + + try { + FeatureToggle.checkHistoricalApiAccessFeatureEnabled(command.getType(), command.getDittoHeaders()); + } catch (final DittoRuntimeException dre) { + getSender().tell(dre, getSelf()); + return; + } + + final CommandStrategy commandStrategy = getCreatedStrategy(); + final EventStrategy eventStrategy = getEventStrategy(); + final ActorRef sender = getSender(); + final ActorRef self = getSelf(); + final long atHistoricalRevision = Optional + .ofNullable(command.getDittoHeaders().get(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey())) + .map(Long::parseLong) + .orElseGet(this::lastSequenceNr); + final Instant atHistoricalTimestamp = Optional + .ofNullable(command.getDittoHeaders().get(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey())) + .map(Instant::parse) + .orElse(Instant.EPOCH); + + loadSnapshot(persistenceId(), SnapshotSelectionCriteria.create( + atHistoricalRevision, + atHistoricalTimestamp.equals(Instant.EPOCH) ? Long.MAX_VALUE : atHistoricalTimestamp.toEpochMilli(), + 0L, + 0L + ), getLatestSnapshotSequenceNumber()); + + final Duration waitTimeout = Duration.ofSeconds(5); + final Cancellable cancellableSnapshotLoadTimeout = + getContext().getSystem().getScheduler().scheduleOnce(waitTimeout, getSelf(), waitTimeout, + getContext().getDispatcher(), getSelf()); + getContext().become(ReceiveBuilder.create() + .match(SnapshotProtocol.LoadSnapshotResult.class, loadSnapshotResult -> + historicalRetrieveHandleLoadSnapshotResult(command, + commandStrategy, + eventStrategy, + sender, + self, + atHistoricalRevision, + atHistoricalTimestamp, + cancellableSnapshotLoadTimeout, + loadSnapshotResult + ) + ) + .match(SnapshotProtocol.LoadSnapshotFailed.class, loadSnapshotFailed -> + log.warning(loadSnapshotFailed.cause(), "Loading snapshot failed") + ) + .matchEquals(waitTimeout, wt -> { + log.withCorrelationId(command) + .warning("Timed out waiting for receiving snapshot result!"); + becomeCreatedOrDeletedHandler(); + unstashAll(); + }) + .matchAny(any -> stash()) + .build()); + } + + private void historicalRetrieveHandleLoadSnapshotResult(final C command, + final CommandStrategy commandStrategy, + final EventStrategy eventStrategy, + final ActorRef sender, + final ActorRef self, + final long atHistoricalRevision, + final Instant atHistoricalTimestamp, + final Cancellable cancellableSnapshotLoadTimeout, + final SnapshotProtocol.LoadSnapshotResult loadSnapshotResult) { + + final Option snapshotEntity = loadSnapshotResult.snapshot() + .map(snapshotAdapter::fromSnapshotStore); + final boolean snapshotIsPresent = snapshotEntity.isDefined(); + + if (snapshotIsPresent || getLatestSnapshotSequenceNumber() == 0) { + final long snapshotEntityRevision = snapshotIsPresent ? + loadSnapshotResult.snapshot().get().metadata().sequenceNr() : 0L; + + final long fromSequenceNr; + if (atHistoricalRevision == snapshotEntityRevision) { + fromSequenceNr = snapshotEntityRevision; + } else { + fromSequenceNr = snapshotEntityRevision + 1; + } + + @Nullable final S entityFromSnapshot = snapshotIsPresent ? snapshotEntity.get() : null; + mongoReadJournal.currentEventsByPersistenceId(persistenceId(), + fromSequenceNr, + atHistoricalRevision + ) + .map(AbstractPersistenceActor::mapJournalEntryToEvent) + .map(journalEntryEvent -> new EntityWithEvent( + eventStrategy.handle((E) journalEntryEvent, entityFromSnapshot, journalEntryEvent.getRevision()), + (E) journalEntryEvent + )) + .takeWhile(entityWithEvent -> { + if (atHistoricalTimestamp.equals(Instant.EPOCH)) { + // no at-historical-timestamp was specified, so take all up to "at-historical-revision": + return true; + } else { + // take while the timestamps of the events are before the specified "at-historical-timestamp": + return entityWithEvent.event.getTimestamp() + .filter(ts -> ts.isBefore(atHistoricalTimestamp)) + .isPresent(); + } + }) + .reduce((ewe1, ewe2) -> new EntityWithEvent( + eventStrategy.handle(ewe2.event, ewe2.entity, ewe2.revision), + ewe2.event + )) + .runWith(Sink.foreach(entityWithEvent -> + commandStrategy.apply(getStrategyContext(), + entityWithEvent.entity, + entityWithEvent.revision, + command + ).accept(new HistoricalResultListener(sender, + entityWithEvent.event.getDittoHeaders())) + ), + getContext().getSystem()); + } else { + if (!atHistoricalTimestamp.equals(Instant.EPOCH)) { + sender.tell(newHistoryNotAccessibleExceptionBuilder(atHistoricalTimestamp).build(), self); + } else { + sender.tell(newHistoryNotAccessibleExceptionBuilder(atHistoricalRevision).build(), self); + } + } + + cancellableSnapshotLoadTimeout.cancel(); + becomeCreatedOrDeletedHandler(); + unstashAll(); + } + /** * Processes a received {@link PingCommand}. * May be overwritten in order to hook into processing ping commands with additional functionality. @@ -358,7 +533,7 @@ protected void persistAndApplyEvent(final E event, final BiConsumer handle } /** - * Allows to modify the passed in {@code event} before {@link #persistEvent(Event, Consumer)} is invoked. + * Allows to modify the passed in {@code event} before {@link #persistEvent(EventsourcedEvent, Consumer)} is invoked. * Overwrite this method and call the super method in order to additionally modify the event before persisting it. * * @param event the event to potentially modify. @@ -452,7 +627,7 @@ private void cancelSnapshot() { } protected void handleByCommandStrategy(final C command) { - handleByStrategy(command, getCreatedStrategy()); + handleByStrategy(command, entity, getCreatedStrategy()); } @SuppressWarnings("unchecked") @@ -462,11 +637,12 @@ private ReceiveBuilder handleByDeletedStrategyReceiveBuilder() { .match(deletedStrategy.getMatchingClass(), deletedStrategy::isDefined, // get the current deletedStrategy during "matching time" to allow implementing classes // to update the strategy during runtime - command -> handleByStrategy(command, (CommandStrategy) getDeletedStrategy())); + command -> handleByStrategy(command, entity, (CommandStrategy) getDeletedStrategy())); } @SuppressWarnings("unchecked") - private > void handleByStrategy(final T command, final CommandStrategy strategy) { + private > void handleByStrategy(final T command, @Nullable final S workEntity, + final CommandStrategy strategy) { log.debug("Handling by strategy: <{}>", command); final var startedSpan = DittoTracing.newPreparedSpan( @@ -481,7 +657,7 @@ private > void handleByStrategy(final T command, final Comm accessCounter++; Result result; try { - result = strategy.apply(getStrategyContext(), entity, getNextRevisionNumber(), (T) tracedCommand); + result = strategy.apply(getStrategyContext(), workEntity, getNextRevisionNumber(), (T) tracedCommand); result.accept(this); } catch (final DittoRuntimeException e) { startedSpan.tagAsFailed(e); @@ -713,6 +889,18 @@ public static Object checkForActivity(final long accessCounter) { return new CheckForActivity(accessCounter); } + private static EventsourcedEvent mapJournalEntryToEvent(final EventEnvelope eventEnvelope) { + + final BsonDocument event = (BsonDocument) eventEnvelope.event(); + final JsonObject eventAsJsonObject = DittoBsonJson.getInstance() + .serialize(event); + + final DittoHeaders dittoHeaders = eventAsJsonObject.getValue(AbstractMongoEventAdapter.HISTORICAL_EVENT_HEADERS) + .map(obj -> DittoHeaders.newBuilder(obj).build()) + .orElseGet(DittoHeaders::empty); + return (EventsourcedEvent) GlobalEventRegistry.getInstance().parse(eventAsJsonObject, dittoHeaders); + } + /** * Check if any command is processed. */ @@ -757,4 +945,71 @@ public String toString() { } + @Immutable + private final class EntityWithEvent { + @Nullable private final S entity; + private final long revision; + private final E event; + + private EntityWithEvent(@Nullable final S entity, final E event) { + this.entity = entity; + this.revision = event.getRevision(); + this.event = event; + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + + "entity=" + entity + + ", revision=" + revision + + ", event=" + event + + ']'; + } + } + + private final class HistoricalResultListener implements ResultVisitor { + + private final ActorRef sender; + private final DittoHeaders historicalDittoHeaders; + + private HistoricalResultListener(final ActorRef sender, final DittoHeaders historicalDittoHeaders) { + this.sender = sender; + this.historicalDittoHeaders = historicalDittoHeaders.toBuilder() + .removeHeader(DittoHeaderDefinition.RESPONSE_REQUIRED.getKey()) + .build(); + } + + @Override + public void onMutation(final Command command, final E event, + final WithDittoHeaders response, + final boolean becomeCreated, final boolean becomeDeleted) { + throw new UnsupportedOperationException("Mutating historical entity not supported."); + } + + @Override + public void onQuery(final Command command, final WithDittoHeaders response) { + if (command.getDittoHeaders().isResponseRequired()) { + final WithDittoHeaders theResponseToSend; + if (response instanceof DittoHeadersSettable dittoHeadersSettable) { + final DittoHeaders queryCommandHeaders = response.getDittoHeaders(); + final DittoHeaders adjustedHeaders = queryCommandHeaders.toBuilder() + .putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), + historicalDittoHeaders.toJson().toString()) + .build(); + theResponseToSend = dittoHeadersSettable.setDittoHeaders(adjustedHeaders); + } else { + theResponseToSend = response; + } + notifySender(sender, theResponseToSend); + } + } + + @Override + public void onError(final DittoRuntimeException error, + final Command errorCausingCommand) { + if (shouldSendResponse(errorCausingCommand.getDittoHeaders())) { + notifySender(sender, error); + } + } + } } diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceSupervisor.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceSupervisor.java index 906df54340a..081e81121bb 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceSupervisor.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceSupervisor.java @@ -14,6 +14,8 @@ import java.text.MessageFormat; import java.time.Duration; +import java.time.Instant; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ThreadLocalRandom; @@ -22,18 +24,24 @@ import javax.annotation.Nullable; +import org.bson.BsonDocument; import org.eclipse.ditto.base.api.commands.sudo.SudoCommand; import org.eclipse.ditto.base.model.entity.id.EntityId; import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.DittoHeadersBuilder; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.WithResource; import org.eclipse.ditto.base.model.signals.WithType; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry; import org.eclipse.ditto.base.service.actors.ShutdownBehaviour; import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOff; import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOffConfig; @@ -49,11 +57,16 @@ import org.eclipse.ditto.internal.utils.metrics.instruments.timer.PreparedTimer; import org.eclipse.ditto.internal.utils.metrics.instruments.timer.StartedTimer; import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces; +import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; +import org.eclipse.ditto.internal.utils.persistence.mongo.DittoBsonJson; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.tracing.DittoTracing; import org.eclipse.ditto.internal.utils.tracing.span.SpanOperationName; +import org.eclipse.ditto.json.JsonObject; import com.typesafe.config.Config; +import akka.NotUsed; import akka.actor.ActorRef; import akka.actor.ActorSystem; import akka.actor.OneForOneStrategy; @@ -69,6 +82,9 @@ import akka.japi.pf.ReceiveBuilder; import akka.pattern.AskTimeoutException; import akka.pattern.Patterns; +import akka.persistence.query.EventEnvelope; +import akka.stream.javadsl.Source; +import akka.stream.javadsl.StreamRefs; /** * Sharded Supervisor of persistent actors. It: @@ -110,10 +126,12 @@ public abstract class AbstractPersistenceSupervisor w.getDittoHeaders().isSudo(), this::forwardDittoSudoToChildIfAvailable) + .match(SubscribeForPersistedEvents.class, this::handleStreamPersistedEvents) .matchAny(matchAnyBehavior) .build(); } + private void handleStreamPersistedEvents(final SubscribeForPersistedEvents subscribeForPersistedEvents) { + + final EntityId commandEntityId = subscribeForPersistedEvents.getEntityId(); + final String persistenceId = commandEntityId.getEntityType() + ":" + commandEntityId; + log.info("Starting to stream persisted events for pid <{}>: {}", persistenceId, subscribeForPersistedEvents); + + final Optional fromHistoricalTimestamp = subscribeForPersistedEvents.getFromHistoricalTimestamp(); + final Optional toHistoricalTimestamp = subscribeForPersistedEvents.getToHistoricalTimestamp(); + final Source startRevisionSource = fromHistoricalTimestamp + .map(fromTs -> mongoReadJournal.getLastSnapshotSequenceNumberBeforeTimestamp(persistenceId, fromTs) + .mergePrioritized( + Source.single(subscribeForPersistedEvents.getFromHistoricalRevision()), + 2, + 1, + false + ) + ) + .orElseGet(() -> Source.single(subscribeForPersistedEvents.getFromHistoricalRevision())); + + final ActorRef sender = getSender(); + askEnforcerChild(subscribeForPersistedEvents) + .whenComplete((enforcedStreamPersistedEvents, throwable) -> { + if (enforcedStreamPersistedEvents instanceof DittoRuntimeException dre) { + log.withCorrelationId(subscribeForPersistedEvents) + .info("Got DittoRuntimeException handling SubscribeForPersistedEvents: " + + "<{}: {}>", dre.getClass().getSimpleName(), dre.getMessage()); + sender.tell(dre, getSelf()); + } else if (null != enforcedStreamPersistedEvents) { + final var sourceRef = startRevisionSource + .flatMapConcat(startRevision -> mongoReadJournal.currentEventsByPersistenceId( + persistenceId, + startRevision, + subscribeForPersistedEvents.getToHistoricalRevision() + )) + .map(eventEnvelope -> + mapJournalEntryToEvent( + (SubscribeForPersistedEvents) enforcedStreamPersistedEvents, eventEnvelope)) + .filter(event -> + fromHistoricalTimestamp.flatMap(instant -> + event.getTimestamp().map(eventTs -> eventTs.isAfter(instant)) + ).orElse(true) + ) + .takeWhile(event -> + toHistoricalTimestamp.flatMap(instant -> + event.getTimestamp().map(eventTs -> eventTs.isBefore(instant)) + ).orElse(true) + ) + .runWith(StreamRefs.sourceRef(), getContext().getSystem()); + sender.tell(sourceRef, getSelf()); + } else if (null != throwable) { + log.withCorrelationId(subscribeForPersistedEvents) + .warning(throwable, "Got throwable: <{}: {}>", throwable.getClass().getSimpleName(), + throwable.getMessage()); + } + }); + } + + private Event mapJournalEntryToEvent(final SubscribeForPersistedEvents enforcedSubscribeForPersistedEvents, + final EventEnvelope eventEnvelope) { + + final BsonDocument event = (BsonDocument) eventEnvelope.event(); + final JsonObject eventAsJsonObject = DittoBsonJson.getInstance() + .serialize(event); + + final DittoHeadersBuilder dittoHeadersBuilder = enforcedSubscribeForPersistedEvents.getDittoHeaders() + .toBuilder(); + eventAsJsonObject.getValue(AbstractMongoEventAdapter.HISTORICAL_EVENT_HEADERS) + .ifPresent(obj -> dittoHeadersBuilder.putHeader( + DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), obj.toString()) + ); + return GlobalEventRegistry.getInstance().parse(eventAsJsonObject, dittoHeadersBuilder.build()); + } + /** * Create a builder for an exception to report unavailability of the entity. * diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/Cleanup.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/Cleanup.java index 461447f692e..08ba16a5357 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/Cleanup.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/Cleanup.java @@ -16,6 +16,7 @@ import static org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal.S_ID; import static org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal.S_SN; +import java.time.Duration; import java.util.List; import java.util.function.Supplier; import java.util.stream.LongStream; @@ -35,6 +36,7 @@ final class Cleanup { private final MongoReadJournal readJournal; private final Materializer materializer; private final Supplier> responsibilitySupplier; + private final Duration historyRetentionDuration; private final int readBatchSize; private final int deleteBatchSize; private final boolean deleteFinalDeletedSnapshot; @@ -42,6 +44,7 @@ final class Cleanup { Cleanup(final MongoReadJournal readJournal, final Materializer materializer, final Supplier> responsibilitySupplier, + final Duration historyRetentionDuration, final int readBatchSize, final int deleteBatchSize, final boolean deleteFinalDeletedSnapshot) { @@ -49,6 +52,7 @@ final class Cleanup { this.readJournal = readJournal; this.materializer = materializer; this.responsibilitySupplier = responsibilitySupplier; + this.historyRetentionDuration = historyRetentionDuration; this.readBatchSize = readBatchSize; this.deleteBatchSize = deleteBatchSize; this.deleteFinalDeletedSnapshot = deleteFinalDeletedSnapshot; @@ -59,8 +63,12 @@ static Cleanup of(final CleanupConfig config, final Materializer materializer, final Supplier> responsibilitySupplier) { - return new Cleanup(readJournal, materializer, responsibilitySupplier, config.getReadsPerQuery(), - config.getWritesPerCredit(), config.shouldDeleteFinalDeletedSnapshot()); + return new Cleanup(readJournal, materializer, responsibilitySupplier, + config.getHistoryRetentionDuration(), + config.getReadsPerQuery(), + config.getWritesPerCredit(), + config.shouldDeleteFinalDeletedSnapshot() + ); } Source, NotUsed> getCleanupStream(final String lowerBound) { @@ -68,7 +76,7 @@ Source, NotUsed> getCleanupStream(final String lo } private Source getSnapshotRevisions(final String lowerBound) { - return readJournal.getNewestSnapshotsAbove(lowerBound, readBatchSize, true, materializer) + return readJournal.getNewestSnapshotsAbove(lowerBound, readBatchSize, true, historyRetentionDuration, materializer) .map(document -> new SnapshotRevision(document.getString(S_ID), document.getLong(S_SN), "DELETED".equals(document.getString(LIFECYCLE)))) @@ -92,7 +100,8 @@ private Source, NotUsed> cleanUpEvents(final Snap } else { final List upperBounds = getSnUpperBoundsPerBatch(minSnOpt.orElseThrow(), sr.sn); return Source.from(upperBounds).map(upperBound -> Source.lazySource(() -> - readJournal.deleteEvents(sr.pid, upperBound - deleteBatchSize + 1, upperBound) + readJournal + .deleteEvents(sr.pid, upperBound - deleteBatchSize + 1, upperBound) .map(result -> new CleanupResult(CleanupResult.Type.EVENTS, sr, result)) ).mapMaterializedValue(ignored -> NotUsed.getInstance())); } diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupConfig.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupConfig.java index ece8620f307..a84469ab36c 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupConfig.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupConfig.java @@ -41,7 +41,7 @@ static CleanupConfig of(final Config config) { * @param config the config values to set. * @return the new cleanup config object. */ - CleanupConfig setAll(final Config config); + CleanupConfig setAll(Config config); /** * Return whether background cleanup is enabled. @@ -50,6 +50,16 @@ static CleanupConfig of(final Config config) { */ boolean isEnabled(); + /** + * Returns the duration of how long to "keep" events and snapshots before being allowed to remove them in scope + * of cleanup. + * If this e.g. is set to 30 days - then effectively an event history of 30 days would be available via the read + * journal. + * + * @return the history retention duration. + */ + Duration getHistoryRetentionDuration(); + /** * Returns quiet period between cleanup streams. * @@ -118,6 +128,12 @@ enum ConfigValue implements KnownConfigValue { */ ENABLED("enabled", true), + /** + * History retention duration. + * Events and snapshots are kept at least that long before cleaning them up from the persistence. + */ + HISTORY_RETENTION_DURATION("history-retention-duration", Duration.ofDays(0L)), + /** * Quiet period. */ diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/DefaultCleanupConfig.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/DefaultCleanupConfig.java index 79b03f78e0a..b2146a62abe 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/DefaultCleanupConfig.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/DefaultCleanupConfig.java @@ -29,6 +29,7 @@ final class DefaultCleanupConfig implements CleanupConfig { static final String CONFIG_PATH = "cleanup"; private final boolean enabled; + private final Duration historyRetentionDuration; private final Duration quietPeriod; private final Duration interval; private final Duration timerThreshold; @@ -38,6 +39,7 @@ final class DefaultCleanupConfig implements CleanupConfig { private final boolean deleteFinalDeletedSnapshot; DefaultCleanupConfig(final boolean enabled, + final Duration historyRetentionDuration, final Duration quietPeriod, final Duration interval, final Duration timerThreshold, @@ -46,6 +48,7 @@ final class DefaultCleanupConfig implements CleanupConfig { final int writesPerCredit, final boolean deleteFinalDeletedSnapshot) { this.enabled = enabled; + this.historyRetentionDuration = historyRetentionDuration; this.quietPeriod = quietPeriod; this.interval = interval; this.timerThreshold = timerThreshold; @@ -57,6 +60,7 @@ final class DefaultCleanupConfig implements CleanupConfig { DefaultCleanupConfig(final ScopedConfig conf) { this.enabled = conf.getBoolean(ConfigValue.ENABLED.getConfigPath()); + this.historyRetentionDuration = conf.getNonNegativeDurationOrThrow(ConfigValue.HISTORY_RETENTION_DURATION); this.quietPeriod = conf.getNonNegativeAndNonZeroDurationOrThrow(ConfigValue.QUIET_PERIOD); this.interval = conf.getNonNegativeAndNonZeroDurationOrThrow(ConfigValue.INTERVAL); this.timerThreshold = conf.getNonNegativeAndNonZeroDurationOrThrow(ConfigValue.TIMER_THRESHOLD); @@ -70,6 +74,7 @@ final class DefaultCleanupConfig implements CleanupConfig { public Config render() { final Map configMap = Map.of( ConfigValue.ENABLED.getConfigPath(), enabled, + ConfigValue.HISTORY_RETENTION_DURATION.getConfigPath(), historyRetentionDuration, ConfigValue.QUIET_PERIOD.getConfigPath(), quietPeriod, ConfigValue.INTERVAL.getConfigPath(), interval, ConfigValue.TIMER_THRESHOLD.getConfigPath(), timerThreshold, @@ -91,6 +96,11 @@ public boolean isEnabled() { return enabled; } + @Override + public Duration getHistoryRetentionDuration() { + return historyRetentionDuration; + } + @Override public Duration getQuietPeriod() { return quietPeriod; @@ -128,9 +138,9 @@ public boolean shouldDeleteFinalDeletedSnapshot() { @Override public boolean equals(final Object o) { - if (o instanceof DefaultCleanupConfig) { - final DefaultCleanupConfig that = (DefaultCleanupConfig) o; + if (o instanceof DefaultCleanupConfig that) { return enabled == that.enabled && + Objects.equals(historyRetentionDuration, that.historyRetentionDuration) && Objects.equals(quietPeriod, that.quietPeriod) && Objects.equals(interval, that.interval) && Objects.equals(timerThreshold, that.timerThreshold) && @@ -145,21 +155,22 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(enabled, quietPeriod, interval, timerThreshold, creditsPerBatch, readsPerQuery, - writesPerCredit, deleteFinalDeletedSnapshot); + return Objects.hash(enabled, historyRetentionDuration, quietPeriod, interval, timerThreshold, creditsPerBatch, + readsPerQuery, writesPerCredit, deleteFinalDeletedSnapshot); } @Override public String toString() { - return getClass().getSimpleName() + - "[enabled=" + enabled + - ",quietPeriod=" + quietPeriod + - ",interval=" + interval + - ",timerThreshold=" + timerThreshold + - ",creditPerBatch=" + creditsPerBatch + - ",readsPerQuery=" + readsPerQuery + - ",writesPerCredit=" + writesPerCredit + - ",deleteFinalDeletedSnapshot=" + deleteFinalDeletedSnapshot + + return getClass().getSimpleName() + "[" + + "enabled=" + enabled + + ", minAgeFromNow=" + historyRetentionDuration + + ", quietPeriod=" + quietPeriod + + ", interval=" + interval + + ", timerThreshold=" + timerThreshold + + ", creditPerBatch=" + creditsPerBatch + + ", readsPerQuery=" + readsPerQuery + + ", writesPerCredit=" + writesPerCredit + + ", deleteFinalDeletedSnapshot=" + deleteFinalDeletedSnapshot + "]"; } diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/AbstractEventStrategies.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/AbstractEventStrategies.java index a9d9439a40d..db2a147718f 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/AbstractEventStrategies.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/AbstractEventStrategies.java @@ -20,7 +20,7 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; -import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +31,7 @@ * @param the type of the entity */ @Immutable -public abstract class AbstractEventStrategies, S> implements EventStrategy { +public abstract class AbstractEventStrategies, S> implements EventStrategy { protected final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/EventStrategy.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/EventStrategy.java index d9a2f7735ac..cebb7a29b10 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/EventStrategy.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/events/EventStrategy.java @@ -14,7 +14,7 @@ import javax.annotation.Nullable; -import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; /** * This interface represents a strategy for handling events in a persistent actor. @@ -23,7 +23,7 @@ * @param the type of the entity */ @FunctionalInterface -public interface EventStrategy, S> { +public interface EventStrategy, S> { /** * Applies an event to an entity. diff --git a/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupTest.java b/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupTest.java index c9ac8417f59..3f74f0ac52a 100644 --- a/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupTest.java +++ b/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CleanupTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -67,7 +68,8 @@ public void emptyStream() { when(mongoReadJournal.getNewestSnapshotsAbove(any(), anyInt(), eq(true), any(), any())) .thenReturn(Source.empty()); - final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), 1, 1, true); + final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), + Duration.ZERO, 1, 1, true); final var result = underTest.getCleanupStream("") .flatMapConcat(x -> x) .runWith(Sink.seq(), materializer) @@ -94,7 +96,8 @@ public void deleteFinalDeletedSnapshot() { invocation.getArgument(1) * 1000L + invocation.getArgument(2) * 10L))) .when(mongoReadJournal).deleteSnapshots(any(), anyLong(), anyLong()); - final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), 1, 4, true); + final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), + Duration.ZERO, 1, 4, true); final var result = underTest.getCleanupStream("") .flatMapConcat(x -> x) @@ -127,7 +130,8 @@ public void excludeFinalDeletedSnapshot() { invocation.getArgument(1) * 1000L + invocation.getArgument(2) * 10L))) .when(mongoReadJournal).deleteSnapshots(any(), anyLong(), anyLong()); - final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), 1, 4, false); + final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), + Duration.ZERO, 1, 4, false); final var result = underTest.getCleanupStream("") .flatMapConcat(x -> x) @@ -168,7 +172,8 @@ public void ignorePidsNotResponsibleFor() { .when(mongoReadJournal).deleteSnapshots(any(), anyLong(), anyLong()); // WHEN: the instance is responsible for 1/3 of the 3 PIDs - final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(2, 3), 1, 4, false); + final var underTest = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(2, 3), + Duration.ZERO, 1, 4, false); final var result = underTest.getCleanupStream("") .flatMapConcat(x -> x) diff --git a/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CreditsTest.java b/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CreditsTest.java index e72be46a6db..e64b21507bf 100644 --- a/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CreditsTest.java +++ b/internal/utils/persistent-actors/src/test/java/org/eclipse/ditto/internal/utils/persistentactors/cleanup/CreditsTest.java @@ -134,7 +134,8 @@ public void onePersistenceWriteAllowedPerCredit() { // mock timer permits 1 batch of credit, after which no credit is given out final var mockTimerResult = new AtomicLong(0L); doAnswer(inv -> mockTimerResult.getAndSet(1001L)).when(mockTimer).getThenReset(); - final var cleanup = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), 1, 4, true); + final var cleanup = new Cleanup(mongoReadJournal, materializer, () -> Pair.create(0, 1), + Duration.ZERO, 1, 4, true); final var underTest = new Credits(getFastCreditConfig(4), mockTimer); final var log = Logging.getLogger(actorSystem, this); @@ -164,7 +165,7 @@ private Pair, TestSubscriber.Probe> material } private static CleanupConfig getFastCreditConfig(final int creditPerBatch) { - return new DefaultCleanupConfig(true, Duration.ZERO, Duration.ofMillis(100), Duration.ofNanos(1000), + return new DefaultCleanupConfig(true, Duration.ZERO, Duration.ZERO, Duration.ofMillis(100), Duration.ofNanos(1000), creditPerBatch, 100, 100, false); } } diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/WithPolicyId.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/WithPolicyId.java new file mode 100644 index 00000000000..904115a7e43 --- /dev/null +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/WithPolicyId.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.policies.model; + +import org.eclipse.ditto.base.model.entity.id.WithEntityId; + +/** + * Implementations of this interface are associated to a {@code Policy} identified by the value + * returned from {@link #getEntityId()}. + * + * @since 3.2.0 + */ +public interface WithPolicyId extends WithEntityId { + + @Override + PolicyId getEntityId(); + +} diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java index 1ed450b33d4..c571ffdd52b 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java @@ -12,24 +12,26 @@ */ package org.eclipse.ditto.policies.model.signals.commands; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.base.model.entity.type.EntityType; import org.eclipse.ditto.base.model.entity.type.WithEntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.policies.model.PolicyConstants; -import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.policies.model.PolicyConstants; +import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.policies.model.WithPolicyId; /** * Aggregates all {@link Command}s which are related to a {@link org.eclipse.ditto.policies.model.Policy}. * * @param the type of the implementing class. */ -public interface PolicyCommand> extends Command, WithEntityType, SignalWithEntityId { +public interface PolicyCommand> extends Command, WithEntityType, WithPolicyId, + SignalWithEntityId { /** * Type Prefix of Policy commands. diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommandResponse.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommandResponse.java index 2933242fc64..16d6d489d46 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommandResponse.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommandResponse.java @@ -12,14 +12,15 @@ */ package org.eclipse.ditto.policies.model.signals.commands; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.policies.model.WithPolicyId; /** * Aggregates all possible responses relating to a given {@link PolicyCommand}. @@ -27,7 +28,7 @@ * @param the type of the implementing class. */ public interface PolicyCommandResponse> extends CommandResponse, - SignalWithEntityId { + WithPolicyId, SignalWithEntityId { /** * Type Prefix of Policy command responses. diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyHistoryNotAccessibleException.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyHistoryNotAccessibleException.java new file mode 100755 index 00000000000..51401af3718 --- /dev/null +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/exceptions/PolicyHistoryNotAccessibleException.java @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.policies.model.signals.commands.exceptions; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.net.URI; +import java.text.MessageFormat; +import java.time.Instant; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.policies.model.PolicyException; +import org.eclipse.ditto.policies.model.PolicyId; + +/** + * Thrown if historical data of the Policy was either not present in Ditto at all or if the requester had insufficient + * permissions to access it. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableException(errorCode = PolicyHistoryNotAccessibleException.ERROR_CODE) +public final class PolicyHistoryNotAccessibleException extends DittoRuntimeException implements PolicyException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "policy.history.notfound"; + + private static final String MESSAGE_TEMPLATE = + "The Policy with ID ''{0}'' at revision ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String MESSAGE_TEMPLATE_TS = + "The Policy with ID ''{0}'' at timestamp ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String DEFAULT_DESCRIPTION = + "Check if the ID of your requested Policy was correct, you have sufficient permissions and ensure that the " + + "asked for revision/timestamp does not exceed the history-retention-duration."; + + private static final long serialVersionUID = 4242422323239998882L; + + private PolicyHistoryNotAccessibleException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + super(ERROR_CODE, HttpStatus.NOT_FOUND, dittoHeaders, message, description, cause, href); + } + + private static String getMessage(final PolicyId policyId, final long revision) { + checkNotNull(policyId, "policyId"); + return MessageFormat.format(MESSAGE_TEMPLATE, String.valueOf(policyId), String.valueOf(revision)); + } + + private static String getMessage(final PolicyId policyId, final Instant timestamp) { + checkNotNull(policyId, "policyId"); + checkNotNull(timestamp, "timestamp"); + return MessageFormat.format(MESSAGE_TEMPLATE_TS, String.valueOf(policyId), timestamp.toString()); + } + + /** + * A mutable builder for a {@code PolicyHistoryNotAccessibleException}. + * + * @param policyId the ID of the policy. + * @param revision the asked for revision of the policy. + * @return the builder. + * @throws NullPointerException if {@code policyId} is {@code null}. + */ + public static Builder newBuilder(final PolicyId policyId, final long revision) { + return new Builder(policyId, revision); + } + + /** + * A mutable builder for a {@code PolicyHistoryNotAccessibleException}. + * + * @param policyId the ID of the policy. + * @param timestamp the asked for timestamp of the policy. + * @return the builder. + * @throws NullPointerException if {@code policyId} is {@code null}. + */ + public static Builder newBuilder(final PolicyId policyId, final Instant timestamp) { + return new Builder(policyId, timestamp); + } + + /** + * Constructs a new {@code PolicyHistoryNotAccessibleException} object with given message. + * + * @param message detail message. This message can be later retrieved by the {@link #getMessage()} method. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new PolicyHistoryNotAccessibleException. + * @throws NullPointerException if {@code dittoHeaders} is {@code null}. + */ + public static PolicyHistoryNotAccessibleException fromMessage(@Nullable final String message, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromMessage(message, dittoHeaders, new Builder()); + } + + /** + * Constructs a new {@code PolicyHistoryNotAccessibleException} object with the exception message extracted from the given + * JSON object. + * + * @param jsonObject the JSON to read the {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new PolicyHistoryNotAccessibleException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static PolicyHistoryNotAccessibleException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyHistoryNotAccessibleException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() { + description(DEFAULT_DESCRIPTION); + } + + private Builder(final PolicyId policyId, final long revision) { + this(); + message(PolicyHistoryNotAccessibleException.getMessage(policyId, revision)); + } + + private Builder(final PolicyId policyId, final Instant timestamp) { + this(); + message(PolicyHistoryNotAccessibleException.getMessage(policyId, timestamp)); + } + + @Override + protected PolicyHistoryNotAccessibleException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new PolicyHistoryNotAccessibleException(dittoHeaders, message, description, cause, href); + } + + } + +} diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/AbstractPolicyEvent.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/AbstractPolicyEvent.java index 3e8cae99661..b94a0103f36 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/AbstractPolicyEvent.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/AbstractPolicyEvent.java @@ -20,8 +20,8 @@ import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.headers.DittoHeaders; -import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.base.model.signals.events.AbstractEventsourcedEvent; +import org.eclipse.ditto.policies.model.PolicyId; /** * Abstract base class of a {@link PolicyEvent}. @@ -57,6 +57,11 @@ protected AbstractPolicyEvent(final String type, this.policyId = policyId; } + @Override + public PolicyId getEntityId() { + return getPolicyEntityId(); + } + @Override public PolicyId getPolicyEntityId() { return policyId; diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java index c818ad61113..9e5c5a50132 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java @@ -14,21 +14,23 @@ import javax.annotation.concurrent.Immutable; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.policies.model.WithPolicyId; /** * Interface for all policy-related events. * * @param the type of the implementing class. */ -public interface PolicyEvent> extends EventsourcedEvent, SignalWithEntityId { +public interface PolicyEvent> extends EventsourcedEvent, WithPolicyId, + SignalWithEntityId { /** * Type Prefix of Policy events. diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsDeletedPartially.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsDeletedPartially.java index 190c8c10e5a..755acbe4696 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsDeletedPartially.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsDeletedPartially.java @@ -66,7 +66,7 @@ public final class SubjectsDeletedPartially extends AbstractPolicyActionEvent JSON_DELETED_SUBJECT_IDS = + public static final JsonFieldDefinition JSON_DELETED_SUBJECT_IDS = JsonFactory.newJsonObjectFieldDefinition("deletedSubjectIds", FieldType.REGULAR, JsonSchemaVersion.V_2); @@ -228,7 +228,13 @@ private static JsonObject deletedSubjectsToJson(final Map> deletedSubjectsFromJson(final JsonObject jsonObject) { + /** + * Transform the passed {@code jsonObject} to a map of deleted subjectIds as expected in the payload of this event. + * + * @param jsonObject the json object to read the modified subjects from. + * @return the map. + */ + public static Map> deletedSubjectsFromJson(final JsonObject jsonObject) { final Map> map = jsonObject.stream() .collect(Collectors.toMap(field -> Label.of(field.getKeyName()), field -> field.getValue().asArray().stream() diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsModifiedPartially.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsModifiedPartially.java index 3588333a549..a1c85153421 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsModifiedPartially.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/SubjectsModifiedPartially.java @@ -67,7 +67,7 @@ public final class SubjectsModifiedPartially extends AbstractPolicyActionEvent JSON_MODIFIED_SUBJECTS = + public static final JsonFieldDefinition JSON_MODIFIED_SUBJECTS = JsonFactory.newJsonObjectFieldDefinition("modifiedSubjects", FieldType.REGULAR, JsonSchemaVersion.V_2); private final Map> modifiedSubjects; @@ -242,7 +242,13 @@ private static JsonObject modifiedSubjectsToJson(final Map> modifiedSubjectsFromJson(final JsonObject jsonObject) { + /** + * Transform the passed {@code jsonObject} to a map of modified subjects as expected in the payload of this event. + * + * @param jsonObject the json object to read the modified subjects from. + * @return the map. + */ + public static Map> modifiedSubjectsFromJson(final JsonObject jsonObject) { final Map> map = jsonObject.stream() .collect(Collectors.toMap(field -> Label.of(field.getKeyName()), field -> subjectsFromJsonWithId(field.getValue().asObject()), diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/DefaultPolicyConfig.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/DefaultPolicyConfig.java index fe125d9d2df..243773d0815 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/DefaultPolicyConfig.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/DefaultPolicyConfig.java @@ -23,7 +23,9 @@ import org.eclipse.ditto.internal.utils.config.ScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultActivityCheckConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultEventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultSnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.CleanupConfig; @@ -40,6 +42,7 @@ public final class DefaultPolicyConfig implements PolicyConfig { private final SupervisorConfig supervisorConfig; private final ActivityCheckConfig activityCheckConfig; private final SnapshotConfig snapshotConfig; + private final EventConfig eventConfig; private final Duration policySubjectExpiryGranularity; private final Duration policySubjectDeletionAnnouncementGranularity; private final String subjectIdResolver; @@ -50,6 +53,7 @@ private DefaultPolicyConfig(final ScopedConfig scopedConfig) { supervisorConfig = DefaultSupervisorConfig.of(scopedConfig); activityCheckConfig = DefaultActivityCheckConfig.of(scopedConfig); snapshotConfig = DefaultSnapshotConfig.of(scopedConfig); + eventConfig = DefaultEventConfig.of(scopedConfig); policySubjectExpiryGranularity = scopedConfig.getNonNegativeDurationOrThrow(PolicyConfigValue.SUBJECT_EXPIRY_GRANULARITY); policySubjectDeletionAnnouncementGranularity = @@ -89,6 +93,11 @@ public SnapshotConfig getSnapshotConfig() { return snapshotConfig; } + @Override + public EventConfig getEventConfig() { + return eventConfig; + } + @Override public Duration getSubjectExpiryGranularity() { return policySubjectExpiryGranularity; @@ -126,6 +135,7 @@ public boolean equals(final Object o) { return Objects.equals(supervisorConfig, that.supervisorConfig) && Objects.equals(activityCheckConfig, that.activityCheckConfig) && Objects.equals(snapshotConfig, that.snapshotConfig) && + Objects.equals(eventConfig, that.eventConfig) && Objects.equals(policySubjectExpiryGranularity, that.policySubjectExpiryGranularity) && Objects.equals(policySubjectDeletionAnnouncementGranularity, that.policySubjectDeletionAnnouncementGranularity) && @@ -136,9 +146,9 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(supervisorConfig, activityCheckConfig, snapshotConfig, policySubjectExpiryGranularity, - policySubjectDeletionAnnouncementGranularity, subjectIdResolver, policyAnnouncementConfig, - cleanupConfig); + return Objects.hash(supervisorConfig, activityCheckConfig, snapshotConfig, eventConfig, + policySubjectExpiryGranularity, policySubjectDeletionAnnouncementGranularity, subjectIdResolver, + policyAnnouncementConfig, cleanupConfig); } @Override @@ -147,6 +157,7 @@ public String toString() { " supervisorConfig=" + supervisorConfig + ", activityCheckConfig=" + activityCheckConfig + ", snapshotConfig=" + snapshotConfig + + ", eventConfig=" + eventConfig + ", policySubjectExpiryGranularity=" + policySubjectExpiryGranularity + ", policySubjectDeletionAnnouncementGranularity=" + policySubjectDeletionAnnouncementGranularity + ", subjectIdResolver=" + subjectIdResolver + diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/PolicyConfig.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/PolicyConfig.java index 83bc6fb910a..7c1c5755d26 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/PolicyConfig.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/common/config/PolicyConfig.java @@ -18,6 +18,7 @@ import org.eclipse.ditto.base.service.config.supervision.WithSupervisorConfig; import org.eclipse.ditto.internal.utils.config.KnownConfigValue; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithSnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.WithCleanupConfig; @@ -29,6 +30,13 @@ public interface PolicyConfig extends WithSupervisorConfig, WithActivityCheckConfig, WithSnapshotConfig, WithCleanupConfig { + /** + * Returns the config of the policy event journal behaviour. + * + * @return the config. + */ + EventConfig getEventConfig(); + /** * Returns the configuration to which duration the {@code expiry} of a {@code Policy Subject} should be rounded up * to. diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcement.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcement.java index 4a92b139c3e..488d41efa20 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcement.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcement.java @@ -18,7 +18,10 @@ import java.util.concurrent.CompletionStage; import org.eclipse.ditto.base.model.auth.AuthorizationContext; +import org.eclipse.ditto.base.model.entity.id.WithEntityId; +import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandToExceptionRegistry; import org.eclipse.ditto.json.JsonFactory; @@ -34,6 +37,7 @@ import org.eclipse.ditto.policies.model.PoliciesResourceType; import org.eclipse.ditto.policies.model.Policy; import org.eclipse.ditto.policies.model.PolicyEntry; +import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.policies.model.ResourceKey; import org.eclipse.ditto.policies.model.enforcers.Enforcer; import org.eclipse.ditto.policies.model.signals.commands.PolicyCommand; @@ -52,7 +56,7 @@ * Authorizes {@link PolicyCommand}s and filters {@link PolicyCommandResponse}s. */ public final class PolicyCommandEnforcement - extends AbstractEnforcementReloaded, PolicyCommandResponse> { + extends AbstractEnforcementReloaded, PolicyCommandResponse> { /** * Json fields that are always shown regardless of authorization. @@ -61,37 +65,38 @@ public final class PolicyCommandEnforcement JsonFactory.newFieldSelector(Policy.JsonFields.ID); @Override - public CompletionStage> authorizeSignal(final PolicyCommand command, + public CompletionStage> authorizeSignal(final Signal signal, final PolicyEnforcer policyEnforcer) { - if (command.getCategory() == Command.Category.QUERY && !command.getDittoHeaders().isResponseRequired()) { + if (signal instanceof Command command && + command.getCategory() == Command.Category.QUERY && !command.getDittoHeaders().isResponseRequired()) { // ignore query command with response-required=false return CompletableFuture.completedStage(null); } final Enforcer enforcer = policyEnforcer.getEnforcer(); - final var policyResourceKey = PoliciesResourceType.policyResource(command.getResourcePath()); - final var authorizationContext = command.getDittoHeaders().getAuthorizationContext(); - final PolicyCommand authorizedCommand; - if (command instanceof CreatePolicy createPolicy) { + final var policyResourceKey = PoliciesResourceType.policyResource(signal.getResourcePath()); + final var authorizationContext = signal.getDittoHeaders().getAuthorizationContext(); + final Signal authorizedCommand; + if (signal instanceof CreatePolicy createPolicy) { authorizedCommand = authorizeCreatePolicy(enforcer, createPolicy, policyResourceKey, authorizationContext); - } else if (command instanceof PolicyActionCommand) { - authorizedCommand = authorizeActionCommand(policyEnforcer, command, policyResourceKey, - authorizationContext).orElseThrow(() -> errorForPolicyCommand(command)); - } else if (command instanceof PolicyModifyCommand) { + } else if (signal instanceof PolicyActionCommand) { + authorizedCommand = authorizeActionCommand(policyEnforcer, signal, policyResourceKey, + authorizationContext).orElseThrow(() -> errorForPolicyCommand(signal)); + } else if (signal instanceof PolicyModifyCommand) { if (hasUnrestrictedWritePermission(enforcer, policyResourceKey, authorizationContext)) { - authorizedCommand = command; + authorizedCommand = signal; } else { - throw errorForPolicyCommand(command); + throw errorForPolicyCommand(signal); } } else { final String permission = Permission.READ; if (enforcer.hasPartialPermissions(policyResourceKey, authorizationContext, permission)) { - authorizedCommand = command; + authorizedCommand = signal; } else { - throw errorForPolicyCommand(command); + throw errorForPolicyCommand(signal); } } @@ -112,8 +117,17 @@ private PolicyCommand authorizeCreatePolicy(final Enforcer enforcer, } @Override - public CompletionStage> authorizeSignalWithMissingEnforcer(final PolicyCommand command) { - throw PolicyNotAccessibleException.newBuilder(command.getEntityId()) + public CompletionStage> authorizeSignalWithMissingEnforcer(final Signal command) { + final PolicyId policyId; + if (command instanceof WithEntityId withEntityId) { + policyId = PolicyId.of(withEntityId.getEntityId()); + } else { + LOGGER.warn("Processed signal which does not have an entityId: {}", command); + throw DittoInternalErrorException.newBuilder() + .dittoHeaders(command.getDittoHeaders()) + .build(); + } + throw PolicyNotAccessibleException.newBuilder(policyId) .dittoHeaders(command.getDittoHeaders()) .build(); } @@ -143,7 +157,7 @@ public CompletionStage> filterResponse(final PolicyComm } @SuppressWarnings("unchecked") - private > Optional authorizeActionCommand( + private > Optional authorizeActionCommand( final PolicyEnforcer enforcer, final T command, final ResourceKey resourceKey, final AuthorizationContext authorizationContext) { @@ -154,7 +168,7 @@ private > Optional authorizeActionCommand( } } - private > Optional authorizeEntryLevelAction(final Enforcer enforcer, + private > Optional authorizeEntryLevelAction(final Enforcer enforcer, final T command, final ResourceKey resourceKey, final AuthorizationContext authorizationContext) { return enforcer.hasUnrestrictedPermissions(resourceKey, authorizationContext, Permission.EXECUTE) ? Optional.of(command) : Optional.empty(); @@ -232,19 +246,32 @@ private JsonObject getJsonViewForPolicyQueryCommandResponse(final JsonObject res /** * Create error due to failing to execute a policy-command in the expected way. * - * @param policyCommand the command. + * @param policySignal the signal. * @return the error. */ - private static DittoRuntimeException errorForPolicyCommand(final PolicyCommand policyCommand) { - final CommandToExceptionRegistry, DittoRuntimeException> registry; - if (policyCommand instanceof PolicyActionCommand) { - registry = PolicyCommandToActionsExceptionRegistry.getInstance(); - } else if (policyCommand instanceof PolicyModifyCommand) { - registry = PolicyCommandToModifyExceptionRegistry.getInstance(); + private static DittoRuntimeException errorForPolicyCommand(final Signal policySignal) { + + if (policySignal instanceof PolicyCommand policyCommand) { + final CommandToExceptionRegistry, DittoRuntimeException> registry; + if (policyCommand instanceof PolicyActionCommand) { + registry = PolicyCommandToActionsExceptionRegistry.getInstance(); + } else if (policyCommand instanceof PolicyModifyCommand) { + registry = PolicyCommandToModifyExceptionRegistry.getInstance(); + } else { + registry = PolicyCommandToAccessExceptionRegistry.getInstance(); + } + return registry.exceptionFrom(policyCommand); + } else if (policySignal instanceof WithEntityId withEntityId) { + return PolicyNotAccessibleException.newBuilder(PolicyId.of(withEntityId.getEntityId())) + .dittoHeaders(policySignal.getDittoHeaders()) + .build(); } else { - registry = PolicyCommandToAccessExceptionRegistry.getInstance(); + LOGGER.error("Received signal for which no DittoRuntimeException due to lack of access " + + "could be determined: {}", policySignal); + return DittoInternalErrorException.newBuilder() + .dittoHeaders(policySignal.getDittoHeaders()) + .build(); } - return registry.exceptionFrom(policyCommand); } /** diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyEnforcerActor.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyEnforcerActor.java index 4b96dd1a42c..8830ec46251 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyEnforcerActor.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyEnforcerActor.java @@ -37,7 +37,7 @@ * {@link PolicyCommandEnforcement}. */ public final class PolicyEnforcerActor extends - AbstractPolicyLoadingEnforcerActor, PolicyCommandResponse, PolicyCommandEnforcement> { + AbstractPolicyLoadingEnforcerActor, PolicyCommandResponse, PolicyCommandEnforcement> { private static final String ENFORCEMENT_DISPATCHER = "enforcement-dispatcher"; diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActor.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActor.java index a64f0c3bb52..51d580dfc28 100755 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActor.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActor.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.policies.service.persistence.actors; +import java.time.Instant; import java.util.Set; import java.util.stream.StreamSupport; @@ -26,6 +27,7 @@ import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceActor; import org.eclipse.ditto.internal.utils.persistentactors.commands.CommandStrategy; import org.eclipse.ditto.internal.utils.persistentactors.commands.DefaultContext; @@ -36,6 +38,7 @@ import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.policies.model.PolicyLifecycle; import org.eclipse.ditto.policies.model.Subjects; +import org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyHistoryNotAccessibleException; import org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyNotAccessibleException; import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.policies.service.common.config.DittoPoliciesConfig; @@ -76,11 +79,12 @@ public final class PolicyPersistenceActor @SuppressWarnings("unused") private PolicyPersistenceActor(final PolicyId policyId, + final MongoReadJournal mongoReadJournal, final ActorRef pubSubMediator, final ActorRef announcementManager, final PolicyConfig policyConfig) { - super(policyId); + super(policyId, mongoReadJournal); this.pubSubMediator = pubSubMediator; this.announcementManager = announcementManager; this.policyConfig = policyConfig; @@ -88,12 +92,13 @@ private PolicyPersistenceActor(final PolicyId policyId, } private PolicyPersistenceActor(final PolicyId policyId, + final MongoReadJournal mongoReadJournal, final ActorRef pubSubMediator, final ActorRef announcementManager, final ActorRef supervisor) { // not possible to call other constructor because "getContext()" is not available as argument of "this()" - super(policyId); + super(policyId, mongoReadJournal); this.pubSubMediator = pubSubMediator; this.announcementManager = announcementManager; this.supervisor = supervisor; @@ -107,25 +112,30 @@ private PolicyPersistenceActor(final PolicyId policyId, * Creates Akka configuration object {@link Props} for this PolicyPersistenceActor. * * @param policyId the ID of the Policy this Actor manages. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the policy. * @param pubSubMediator the PubSub mediator actor. * @param announcementManager manager of policy announcements. * @param policyConfig the policy config. * @return the Akka configuration Props object */ public static Props props(final PolicyId policyId, + final MongoReadJournal mongoReadJournal, final ActorRef pubSubMediator, final ActorRef announcementManager, final PolicyConfig policyConfig) { - return Props.create(PolicyPersistenceActor.class, policyId, pubSubMediator, announcementManager, policyConfig); + return Props.create(PolicyPersistenceActor.class, policyId, mongoReadJournal, pubSubMediator, + announcementManager, policyConfig); } static Props propsForTests(final PolicyId policyId, + final MongoReadJournal mongoReadJournal, final ActorRef pubSubMediator, final ActorRef announcementManager, final ActorSystem actorSystem) { - return Props.create(PolicyPersistenceActor.class, policyId, pubSubMediator, announcementManager, + return Props.create(PolicyPersistenceActor.class, policyId, mongoReadJournal, pubSubMediator, + announcementManager, actorSystem.deadLetters()); } @@ -189,6 +199,16 @@ protected DittoRuntimeExceptionBuilder newNotAccessibleExceptionBuilder() { return PolicyNotAccessibleException.newBuilder(entityId); } + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final long revision) { + return PolicyHistoryNotAccessibleException.newBuilder(entityId, revision); + } + + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final Instant timestamp) { + return PolicyHistoryNotAccessibleException.newBuilder(entityId, timestamp); + } + @Override protected void publishEvent(@Nullable final Policy previousEntity, final PolicyEvent event) { diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActor.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActor.java index bf5869071e2..b089a5ff9f5 100755 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActor.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActor.java @@ -22,6 +22,7 @@ import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOffConfig; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.policies.enforcement.PolicyEnforcerProvider; @@ -58,9 +59,10 @@ public final class PolicySupervisorActor extends AbstractPersistenceSupervisor

> policyAnnouncementPub, @Nullable final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { - super(blockedNamespaces, DEFAULT_LOCAL_ASK_TIMEOUT); + super(blockedNamespaces, mongoReadJournal, DEFAULT_LOCAL_ASK_TIMEOUT); this.policyEnforcerProvider = policyEnforcerProvider; this.pubSubMediator = pubSubMediator; final DittoPoliciesConfig policiesConfig = DittoPoliciesConfig.of( @@ -88,15 +90,17 @@ private PolicySupervisorActor(final ActorRef pubSubMediator, * @param policyAnnouncementPub publisher interface of policy announcements. * @param blockedNamespaces the blocked namespaces functionality to retrieve/subscribe for blocked namespaces. * @param policyEnforcerProvider used to load the policy enforcer to authorize commands. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the policy. * @return the {@link Props} to create this actor. */ public static Props props(final ActorRef pubSubMediator, final DistributedPub> policyAnnouncementPub, @Nullable final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { - return Props.create(PolicySupervisorActor.class, () -> new PolicySupervisorActor(pubSubMediator, - policyAnnouncementPub, blockedNamespaces, policyEnforcerProvider)); + return Props.create(PolicySupervisorActor.class, pubSubMediator, + policyAnnouncementPub, blockedNamespaces, policyEnforcerProvider, mongoReadJournal); } @Override @@ -106,7 +110,8 @@ protected PolicyId getEntityId() throws Exception { @Override protected Props getPersistenceActorProps(final PolicyId entityId) { - return PolicyPersistenceActor.props(entityId, pubSubMediator, announcementManager, policyConfig); + return PolicyPersistenceActor.props(entityId, mongoReadJournal, pubSubMediator, announcementManager, + policyConfig); } @Override diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/events/PolicyEventStrategies.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/events/PolicyEventStrategies.java index 10afaeddbec..18fae3502a5 100755 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/events/PolicyEventStrategies.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/actors/strategies/events/PolicyEventStrategies.java @@ -38,7 +38,7 @@ import org.eclipse.ditto.policies.model.signals.events.SubjectsModifiedPartially; /** - * Holds all {@link org.eclipse.ditto.policies.model.signals.events.PolicyEvent} strategies. + * Holds all {@link PolicyEvent} strategies. */ public final class PolicyEventStrategies extends AbstractEventStrategies, Policy> { diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/AbstractPolicyMongoEventAdapter.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/AbstractPolicyMongoEventAdapter.java index 822b0926561..b296cdb1a62 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/AbstractPolicyMongoEventAdapter.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/AbstractPolicyMongoEventAdapter.java @@ -13,16 +13,17 @@ package org.eclipse.ditto.policies.service.persistence.serializer; -import javax.annotation.Nullable; - -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; -import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry; +import org.eclipse.ditto.base.service.config.DittoServiceConfig; +import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.policies.service.common.config.DefaultPolicyConfig; import akka.actor.ExtendedActorSystem; @@ -36,8 +37,10 @@ public abstract class AbstractPolicyMongoEventAdapter extends AbstractMongoEvent JsonFactory.newJsonObjectFieldDefinition("policy/entries", FieldType.SPECIAL, JsonSchemaVersion.V_2); - protected AbstractPolicyMongoEventAdapter(@Nullable final ExtendedActorSystem system) { - super(system, GlobalEventRegistry.getInstance()); + protected AbstractPolicyMongoEventAdapter(final ExtendedActorSystem system) { + super(system, GlobalEventRegistry.getInstance(), DefaultPolicyConfig.of( + DittoServiceConfig.of(DefaultScopedConfig.dittoScoped(system.settings().config()), "policies")) + .getEventConfig()); } } diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/DefaultPolicyMongoEventAdapter.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/DefaultPolicyMongoEventAdapter.java index 59a0c3a1b19..ec5fec64cd5 100644 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/DefaultPolicyMongoEventAdapter.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/persistence/serializer/DefaultPolicyMongoEventAdapter.java @@ -12,8 +12,6 @@ */ package org.eclipse.ditto.policies.service.persistence.serializer; -import javax.annotation.Nullable; - import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import akka.actor.ExtendedActorSystem; @@ -24,7 +22,7 @@ */ public final class DefaultPolicyMongoEventAdapter extends AbstractPolicyMongoEventAdapter { - public DefaultPolicyMongoEventAdapter(@Nullable final ExtendedActorSystem system) { + public DefaultPolicyMongoEventAdapter(final ExtendedActorSystem system) { super(system); } diff --git a/policies/service/src/main/java/org/eclipse/ditto/policies/service/starter/PoliciesRootActor.java b/policies/service/src/main/java/org/eclipse/ditto/policies/service/starter/PoliciesRootActor.java index 5db4b1f3105..f1ea2413c21 100755 --- a/policies/service/src/main/java/org/eclipse/ditto/policies/service/starter/PoliciesRootActor.java +++ b/policies/service/src/main/java/org/eclipse/ditto/policies/service/starter/PoliciesRootActor.java @@ -88,15 +88,16 @@ private PoliciesRootActor(final PoliciesConfig policiesConfig, final ActorRef pu BlockedNamespacesUpdater.ACTOR_NAME, blockedNamespacesUpdaterProps); final PolicyEnforcerProvider policyEnforcerProvider = PolicyEnforcerProviderExtension.get(actorSystem).getPolicyEnforcerProvider(); + final var mongoReadJournal = MongoReadJournal.newInstance(actorSystem); + final var policySupervisorProps = getPolicySupervisorActorProps(pubSubMediator, policyAnnouncementPub, blockedNamespaces, - policyEnforcerProvider); + policyEnforcerProvider, mongoReadJournal); final ActorRef policiesShardRegion = ShardRegionCreator.start(actorSystem, PoliciesMessagingConstants.SHARD_REGION, policySupervisorProps, policiesConfig.getClusterConfig().getNumberOfShards(), CLUSTER_ROLE); - final var mongoReadJournal = MongoReadJournal.newInstance(actorSystem); startClusterSingletonActor( PersistencePingActor.props(policiesShardRegion, policiesConfig.getPingConfig(), mongoReadJournal), PersistencePingActor.ACTOR_NAME); @@ -133,10 +134,11 @@ private PoliciesRootActor(final PoliciesConfig policiesConfig, final ActorRef pu private static Props getPolicySupervisorActorProps(final ActorRef pubSubMediator, final DistributedPub> policyAnnouncementPub, final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { return PolicySupervisorActor.props(pubSubMediator, policyAnnouncementPub, blockedNamespaces, - policyEnforcerProvider); + policyEnforcerProvider, mongoReadJournal); } /** diff --git a/policies/service/src/main/resources/policies.conf b/policies/service/src/main/resources/policies.conf index 0023efa356b..0f1a79e8065 100755 --- a/policies/service/src/main/resources/policies.conf +++ b/policies/service/src/main/resources/policies.conf @@ -65,6 +65,16 @@ ditto { threshold = ${?POLICY_SNAPSHOT_THRESHOLD} # may be overridden with this environment variable } + event { + # define the DittoHeaders to persist when persisting events to the journal + # those can e.g. be retrieved as additional "audit log" information when accessing a historical policy revision + historical-headers-to-persist = [ + #"ditto-originator" # who (user-subject/connection-pre-auth-subject) issued the event + #"correlation-id" + ] + historical-headers-to-persist = ${?POLICY_EVENT_HISTORICAL_HEADERS_TO_PERSIST} + } + supervisor { exponential-backoff { min = 1s @@ -102,27 +112,62 @@ ditto { } cleanup { + # enabled configures whether background cleanup is enabled or not + # If enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process enabled = true enabled = ${?CLEANUP_ENABLED} + # history-retention-duration configures the duration of how long to "keep" events and snapshots before being + # allowed to remove them in scope of cleanup. + # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read + # journal. + history-retention-duration = 0d + history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} + + # quiet-period defines how long to stay in a state where the background cleanup is not yet started + # Applies after: + # - starting the service + # - each "completed" background cleanup run (all entities were cleaned up) quiet-period = 5m quiet-period = ${?CLEANUP_QUIET_PERIOD} + # interval configures how often a "credit decision" is made. + # The background cleanup works with a credit system and does only generate new "cleanup credits" if the MongoDB + # currently has capacity to do cleanups. interval = 10s interval = ${?CLEANUP_INTERVAL} + # timer-threshold configures the maximum database latency to give out credit for cleanup actions. + # If write operations to the MongoDB within the last `interval` had a `max` value greater to the configured + # threshold, no new cleanup credits will be issued for the next `interval`. + # Which throttles cleanup when MongoDB is currently under heavy (write) load. timer-threshold = 150ms timer-threshold = ${?CLEANUP_TIMER_THRESHOLD} + # credits-per-batch configures how many "cleanup credits" should be generated per `interval` as long as the + # write operations to the MongoDB are less than the configured `timer-threshold`. + # Limits the rate of cleanup actions to this many per credit decision interval. + # One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`. credits-per-batch = 3 credits-per-batch = ${?CLEANUP_CREDITS_PER_BATCH} + # reads-per-query configures the number of snapshots to scan per MongoDB query. + # Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned + # with the amount of "cleanup credits" issued per `interval` - in order to avoid long running queries. reads-per-query = 100 reads-per-query = ${?CLEANUP_READS_PER_QUERY} + # writes-per-credit configures the number of documents to delete for each credit. + # If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead + # to 10 delete operations performed against MongoDB. writes-per-credit = 100 writes-per-credit = ${?CLEANUP_WRITES_PER_CREDIT} + # delete-final-deleted-snapshot configures whether for a deleted entity, the final snapshot (containing the + # "deleted" information) should be deleted or not. + # If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with + # a revision number 1 higher than the previously deleted entity. If the final snapshot is deleted as well, + # recreation of an entity with the same ID will lead to revisionNumber=1 after its recreation. delete-final-deleted-snapshot = false delete-final-deleted-snapshot = ${?CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT} } @@ -219,6 +264,18 @@ akka-contrib-mongodb-persistence-policies-journal { } } +akka-contrib-mongodb-persistence-policies-journal-read { + class = "akka.contrib.persistence.mongodb.MongoReadJournal" + plugin-dispatcher = "policy-journal-persistence-dispatcher" + + overrides { + journal-collection = "policies_journal" + journal-index = "policies_journal_index" + realtime-collection = "policies_realtime" + metadata-collection = "policies_metadata" + } +} + akka-contrib-mongodb-persistence-policies-snapshots { class = "akka.contrib.persistence.mongodb.MongoSnapshots" plugin-dispatcher = "policy-snaps-persistence-dispatcher" diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcementTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcementTest.java index 2f367ad8b55..a3a48efb88b 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcementTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/enforcement/PolicyCommandEnforcementTest.java @@ -42,6 +42,7 @@ import org.eclipse.ditto.base.service.actors.ShutdownBehaviour; import org.eclipse.ditto.base.service.config.supervision.ExponentialBackOffConfig; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.json.JsonObject; @@ -785,7 +786,7 @@ private class MockPolicyPersistenceSupervisor private MockPolicyPersistenceSupervisor(final ActorRef pubSubMediator, final ActorRef policyPersistenceActor, final PolicyEnforcerProvider policyEnforcerProvider) { - super(policyPersistenceActor, null, null, Duration.ofSeconds(5)); + super(policyPersistenceActor, null, null, Mockito.mock(MongoReadJournal.class), Duration.ofSeconds(5)); this.pubSubMediator = pubSubMediator; this.policyEnforcerProvider = policyEnforcerProvider; } diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorSnapshottingTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorSnapshottingTest.java index dc0369b511f..10ccdd21ee3 100755 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorSnapshottingTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorSnapshottingTest.java @@ -32,6 +32,7 @@ import org.eclipse.ditto.base.model.signals.events.AbstractEventsourcedEvent; import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; import org.eclipse.ditto.internal.utils.persistence.mongo.DittoBsonJson; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.internal.utils.test.Retry; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; @@ -415,7 +416,8 @@ private static void assertPolicyInJournal(final Policy actualPolicy, final Polic } private ActorRef createPersistenceActorFor(final PolicyId policyId) { - final Props props = PolicyPersistenceActor.propsForTests(policyId, pubSubMediator, actorSystem.deadLetters(), + final Props props = PolicyPersistenceActor.propsForTests(policyId, Mockito.mock(MongoReadJournal.class), + pubSubMediator, actorSystem.deadLetters(), actorSystem); return actorSystem.actorOf(props); } diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorTest.java index 655ef28f243..01a541a78f8 100755 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceActorTest.java @@ -48,6 +48,7 @@ import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.internal.utils.cluster.ShardRegionExtractor; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceActor; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; @@ -1389,7 +1390,8 @@ public void ensureExpiredSubjectIsRemovedDuringRecovery() throws InterruptedExce ClusterShardingSettings.apply(actorSystem).withRole("policies"); final var box = new AtomicReference(); final ActorRef announcementManager = createAnnouncementManager(policyId, box::get); - final Props props = PolicyPersistenceActor.propsForTests(policyId, pubSubMediator, announcementManager, + final Props props = PolicyPersistenceActor.propsForTests(policyId, Mockito.mock(MongoReadJournal.class), + pubSubMediator, announcementManager, actorSystem); final Cluster cluster = Cluster.get(actorSystem); cluster.join(cluster.selfAddress()); @@ -1640,7 +1642,8 @@ public void checkForActivityOfNonexistentPolicy() { final var box = new AtomicReference(); final ActorRef announcementManager = createAnnouncementManager(policyId, box::get); final Props persistentActorProps = - PolicyPersistenceActor.propsForTests(policyId, pubSubMediator, announcementManager, actorSystem); + PolicyPersistenceActor.propsForTests(policyId, Mockito.mock(MongoReadJournal.class), + pubSubMediator, announcementManager, actorSystem); final TestProbe errorsProbe = TestProbe.apply(actorSystem); @@ -1755,7 +1758,8 @@ private ActorRef createPersistenceActorFor(final TestKit testKit, final Policy p private ActorRef createPersistenceActorFor(final TestKit testKit, final PolicyId policyId) { final var box = new AtomicReference(); final ActorRef announcementManager = createAnnouncementManager(policyId, box::get); - final Props props = PolicyPersistenceActor.propsForTests(policyId, pubSubMediator, announcementManager, + final Props props = PolicyPersistenceActor.propsForTests(policyId, Mockito.mock(MongoReadJournal.class), + pubSubMediator, announcementManager, actorSystem); final var persistenceActor = testKit.watch(testKit.childActorOf(props)); box.set(persistenceActor); diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceOperationsActorIT.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceOperationsActorIT.java index 542a6ab07ea..8ee2759eec5 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceOperationsActorIT.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicyPersistenceOperationsActorIT.java @@ -28,6 +28,7 @@ import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; import org.eclipse.ditto.internal.utils.persistence.mongo.MongoClientWrapper; import org.eclipse.ditto.internal.utils.persistence.mongo.ops.eventsource.MongoEventSourceITAssertions; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistence.operations.EntityPersistenceOperations; import org.eclipse.ditto.internal.utils.persistence.operations.NamespacePersistenceOperations; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; @@ -236,9 +237,8 @@ private ActorRef startActorUnderTest(final ActorSystem actorSystem, final ActorR @Override protected ActorRef startEntityActor(final ActorSystem system, final ActorRef pubSubMediator, final PolicyId id) { final Props props = - PolicySupervisorActor.props(pubSubMediator, Mockito.mock(DistributedPub.class), null, Mockito.mock( - PolicyEnforcerProvider.class)); - + PolicySupervisorActor.props(pubSubMediator, Mockito.mock(DistributedPub.class), null, + Mockito.mock(PolicyEnforcerProvider.class), Mockito.mock(MongoReadJournal.class)); return system.actorOf(props, id.toString()); } diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActorTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActorTest.java index 252bf13c47c..12f659dcd31 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActorTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/persistence/actors/PolicySupervisorActorTest.java @@ -20,6 +20,7 @@ import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.internal.utils.cluster.StopShardedActor; import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.policies.enforcement.PolicyEnforcer; @@ -35,6 +36,7 @@ import org.junit.ClassRule; import org.junit.Test; import org.mockito.Mock; +import org.mockito.Mockito; import akka.stream.Attributes; import akka.testkit.TestProbe; @@ -80,7 +82,8 @@ public void setup() { public void stopNonexistentPolicy() { new TestKit(actorSystem) {{ final PolicyId policyId = PolicyId.of("test.ns", "stopNonexistentPolicy"); - final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, policyEnforcerProvider); + final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, + policyEnforcerProvider, Mockito.mock(MongoReadJournal.class)); final var underTest = watch(childActorOf(props, policyId.toString())); underTest.tell(new StopShardedActor(), getRef()); expectTerminated(underTest); @@ -91,7 +94,8 @@ public void stopNonexistentPolicy() { public void stopAfterRetrievingNonexistentPolicy() { new TestKit(actorSystem) {{ final PolicyId policyId = PolicyId.of("test.ns", "retrieveNonexistentPolicy"); - final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, policyEnforcerProvider); + final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, + policyEnforcerProvider, Mockito.mock(MongoReadJournal.class)); final var underTest = watch(childActorOf(props, policyId.toString())); final var probe = TestProbe.apply(actorSystem); final var retrievePolicy = RetrievePolicy.of(policyId, DittoHeaders.empty()); @@ -111,7 +115,8 @@ public void stopAfterRetrievingExistingPolicy() { { final var policy = createPolicyWithRandomId(); final var policyId = policy.getEntityId().orElseThrow(); - final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, policyEnforcerProvider); + final var props = PolicySupervisorActor.props(pubSubMediator, pub, blockedNamespaces, + policyEnforcerProvider, Mockito.mock(MongoReadJournal.class)); final var underTest = watch(childActorOf(props, policyId.toString())); final var probe = TestProbe.apply(actorSystem); diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalCommandRegistryTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalCommandRegistryTest.java index 3360e4699d1..e2c9f1beb87 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalCommandRegistryTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalCommandRegistryTest.java @@ -17,6 +17,7 @@ import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; import org.eclipse.ditto.internal.models.streaming.SudoStreamPids; import org.eclipse.ditto.internal.utils.health.RetrieveHealth; @@ -47,7 +48,8 @@ public PoliciesServiceGlobalCommandRegistryTest() { PurgeEntities.class, PublishSignal.class, ModifyPolicyImports.class, - ModifySplitBrainResolver.class + ModifySplitBrainResolver.class, + SubscribeForPersistedEvents.class ); } } diff --git a/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalEventRegistryTest.java b/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalEventRegistryTest.java index a1bcd6a5385..2d73db04b1d 100644 --- a/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalEventRegistryTest.java +++ b/policies/service/src/test/java/org/eclipse/ditto/policies/service/starter/PoliciesServiceGlobalEventRegistryTest.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.policies.service.starter; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; import org.eclipse.ditto.internal.utils.persistentactors.EmptyEvent; import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases; import org.eclipse.ditto.policies.model.signals.events.ResourceDeleted; @@ -21,7 +22,8 @@ public final class PoliciesServiceGlobalEventRegistryTest extends GlobalEventReg public PoliciesServiceGlobalEventRegistryTest() { super( ResourceDeleted.class, - EmptyEvent.class + EmptyEvent.class, + StreamingSubscriptionComplete.class ); } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutablePayload.java b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutablePayload.java index 291c49ca3e3..fa41e6de4f7 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutablePayload.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutablePayload.java @@ -244,7 +244,7 @@ public PayloadBuilder withPath(@Nullable final JsonPointer path) { } @Override - public ImmutablePayloadBuilder withValue(final JsonValue value) { + public ImmutablePayloadBuilder withValue(@Nullable final JsonValue value) { this.value = value; return this; } @@ -280,14 +280,14 @@ public ImmutablePayloadBuilder withMetadata(@Nullable final Metadata metadata) { } @Override - public ImmutablePayloadBuilder withFields(final JsonFieldSelector fields) { + public ImmutablePayloadBuilder withFields(@Nullable final JsonFieldSelector fields) { this.fields = fields; return this; } @Override - public ImmutablePayloadBuilder withFields(final String fields) { - this.fields = JsonFieldSelector.newInstance(fields); + public ImmutablePayloadBuilder withFields(@Nullable final String fields) { + this.fields = null != fields ? JsonFieldSelector.newInstance(fields) : null; return this; } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableTopicPath.java b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableTopicPath.java index 65b193e07e6..8bcc89c638d 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableTopicPath.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/ImmutableTopicPath.java @@ -30,6 +30,7 @@ import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.NotThreadSafe; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonKey; import org.eclipse.ditto.json.JsonPointer; @@ -48,6 +49,7 @@ final class ImmutableTopicPath implements TopicPath { private final Criterion criterion; @Nullable private final Action action; @Nullable private final SearchAction searchAction; + @Nullable private final StreamingAction streamingAction; @Nullable private final String subject; private ImmutableTopicPath(final Builder builder) { @@ -58,6 +60,7 @@ private ImmutableTopicPath(final Builder builder) { criterion = builder.criterion; action = builder.action; searchAction = builder.searchAction; + streamingAction = builder.streamingAction; subject = builder.subject; } @@ -139,6 +142,11 @@ public Optional getSearchAction() { return Optional.ofNullable(searchAction); } + @Override + public Optional getStreamingAction() { + return Optional.ofNullable(streamingAction); + } + @Override public Optional getSubject() { return Optional.ofNullable(subject); @@ -159,6 +167,7 @@ public String getPath() { .add(criterion.getName()) .add(getStringOrNull(action)) .add(getStringOrNull(searchAction)) + .add(getStringOrNull(streamingAction)) .add(getStringOrNull(subject)) .build(); return pathPartStream.filter(Objects::nonNull).collect(Collectors.joining(PATH_DELIMITER)); @@ -212,12 +221,14 @@ public boolean equals(final Object o) { criterion == that.criterion && Objects.equals(action, that.action) && Objects.equals(searchAction, that.searchAction) && + Objects.equals(streamingAction, that.streamingAction) && Objects.equals(subject, that.subject); } @Override public int hashCode() { - return Objects.hash(namespace, name, group, channel, criterion, action, searchAction, subject); + return Objects.hash(namespace, name, group, channel, criterion, action, searchAction, streamingAction, + subject); } @Override @@ -230,6 +241,7 @@ public String toString() { ", criterion=" + criterion + ", action=" + action + ", searchAction=" + searchAction + + ", streamingAction=" + streamingAction + ", subject=" + subject + ", path=" + getPath() + "]"; @@ -241,7 +253,8 @@ public String toString() { @NotThreadSafe private static final class Builder implements TopicPathBuilder, MessagesTopicPathBuilder, EventsTopicPathBuilder, CommandsTopicPathBuilder, - AcknowledgementTopicPathBuilder, SearchTopicPathBuilder, AnnouncementsTopicPathBuilder { + AcknowledgementTopicPathBuilder, SearchTopicPathBuilder, AnnouncementsTopicPathBuilder, + StreamingTopicPathBuilder { private final String namespace; private final String name; @@ -251,6 +264,7 @@ private static final class Builder private Criterion criterion; @Nullable private Action action; @Nullable private SearchAction searchAction; + @Nullable private StreamingAction streamingAction; @Nullable private String subject; private Builder(final String namespace, final String name) { @@ -261,6 +275,7 @@ private Builder(final String namespace, final String name) { criterion = null; action = null; searchAction = null; + streamingAction = null; subject = null; } @@ -318,6 +333,12 @@ public AnnouncementsTopicPathBuilder announcements() { return this; } + @Override + public StreamingTopicPathBuilder streaming() { + criterion = Criterion.STREAMING; + return this; + } + @Override public EventsTopicPathBuilder events() { criterion = Criterion.EVENTS; @@ -378,45 +399,83 @@ public TopicPathBuildable subscribe() { return this; } + @Override + public TopicPathBuildable subscribe(final String subscribingCommandName) { + if (subscribingCommandName.equals(SubscribeForPersistedEvents.NAME)) { + streamingAction = StreamingAction.SUBSCRIBE_FOR_PERSISTED_EVENTS; + } else { + throw UnknownCommandException.newBuilder(subscribingCommandName).build(); + } + return this; + } + @Override public TopicPathBuildable cancel() { - searchAction = SearchAction.CANCEL; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.CANCEL; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.CANCEL; + } return this; } @Override public TopicPathBuildable request() { - searchAction = SearchAction.REQUEST; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.REQUEST; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.REQUEST; + } return this; } @Override public TopicPathBuildable complete() { - searchAction = SearchAction.COMPLETE; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.COMPLETE; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.COMPLETE; + } return this; } @Override public TopicPathBuildable failed() { - searchAction = SearchAction.FAILED; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.FAILED; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.FAILED; + } return this; } @Override public TopicPathBuildable hasNext() { - searchAction = SearchAction.NEXT; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.NEXT; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.NEXT; + } return this; } @Override public EventsTopicPathBuilder generated() { - searchAction = SearchAction.GENERATED; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.GENERATED; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.GENERATED; + } return this; } @Override public TopicPathBuildable error() { - searchAction = SearchAction.ERROR; + if (criterion == Criterion.SEARCH) { + searchAction = SearchAction.ERROR; + } else if (criterion == Criterion.STREAMING) { + streamingAction = StreamingAction.ERROR; + } return this; } @@ -521,6 +580,9 @@ public ImmutableTopicPath get() { case SEARCH: topicPathBuilder.searchAction = tryToGetSearchActionForName(tryToGetSearchActionName()); break; + case STREAMING: + topicPathBuilder.streamingAction = tryToGetStreamingActionForName(tryToGetSearchActionName()); + break; case ERRORS: break; case MESSAGES: @@ -656,6 +718,13 @@ private SearchAction tryToGetSearchActionForName(final String searchActionName) .build()); } + private StreamingAction tryToGetStreamingActionForName(final String streamingActionName) { + return StreamingAction.forName(streamingActionName) + .orElseThrow(() -> UnknownTopicPathException.newBuilder(topicPathString) + .description(MessageFormat.format("Streaming action name <{0}> is unknown.", streamingActionName)) + .build()); + } + @Nullable private String getSubjectOrNull() { final String subject = String.join(TopicPath.PATH_DELIMITER, topicPathParts); diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/ProtocolFactory.java b/protocol/src/main/java/org/eclipse/ditto/protocol/ProtocolFactory.java index 3e4244a46be..021c0a0c142 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/ProtocolFactory.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/ProtocolFactory.java @@ -19,9 +19,11 @@ import java.util.stream.Collectors; import org.eclipse.ditto.base.model.common.DittoConstants; +import org.eclipse.ditto.base.model.entity.id.EntityId; import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.contenttype.ContentType; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonPointer; @@ -126,6 +128,38 @@ public static TopicPathBuilder newTopicPathBuilder(final NamespacedEntityId enti return result; } + /** + * Returns a new {@code TopicPathBuilder} for the specified {@link EntityId}. + * The namespace and name part of the {@code TopicPath} will pe parsed from the entity ID and set in the builder. + * + * @param entityId the ID. + * @return the builder. + * @throws NullPointerException if {@code entityId} is {@code null}. + * @throws org.eclipse.ditto.things.model.ThingIdInvalidException if {@code entityId} is not in the expected + * format. + * @since 3.2.0 + */ + public static TopicPathBuilder newTopicPathBuilder(final EntityId entityId) { + checkNotNull(entityId, "entityId"); + final TopicPathBuilder result; + if (entityId instanceof NamespacedEntityId) { + final String namespace = ((NamespacedEntityId) entityId).getNamespace(); + final String name = ((NamespacedEntityId) entityId).getName(); + result = ImmutableTopicPath.newBuilder(namespace, name); + } else { + result = ProtocolFactory.newTopicPathBuilderFromName(entityId.toString()); + } + + if (entityId.getEntityType().equals(ThingConstants.ENTITY_TYPE)) { + return result.things(); + } else if (entityId.getEntityType().equals(PolicyConstants.ENTITY_TYPE)) { + return result.policies(); + } else if (entityId.getEntityType().equals(ConnectivityConstants.ENTITY_TYPE)) { + return result.connections(); + } + return result; + } + /** * Returns a new {@code TopicPathBuilder} for the specified {@link PolicyId}. * The namespace and name part of the {@code TopicPath} will pe parsed from the {@code PolicyId} and set in the diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/StreamingTopicPathBuilder.java b/protocol/src/main/java/org/eclipse/ditto/protocol/StreamingTopicPathBuilder.java new file mode 100644 index 00000000000..5c1af097bbd --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/StreamingTopicPathBuilder.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol; + +/** + * Builder to create a topic path for streaming commands. + * + * @since 3.2.0 + */ +public interface StreamingTopicPathBuilder extends TopicPathBuildable { + + /** + * Sets the {@code Action} of this builder to the passed {@code subscribingCommandName}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable subscribe(String subscribingCommandName); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#CANCEL}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable cancel(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#REQUEST}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable request(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#COMPLETE}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable complete(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#FAILED}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable failed(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#NEXT}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable hasNext(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#GENERATED}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable generated(); + + /** + * Sets the {@code Action} of this builder to {@link TopicPath.StreamingAction#ERROR}. A previously set action is replaced. + * + * @return this builder to allow method chaining. + */ + TopicPathBuildable error(); + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPath.java b/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPath.java index 3af44dd32af..6fa1c1f0906 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPath.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPath.java @@ -19,6 +19,14 @@ import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionCreated; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionHasNext; +import org.eclipse.ditto.connectivity.model.ConnectionId; import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.policies.model.PolicyConstants; import org.eclipse.ditto.policies.model.PolicyId; @@ -68,6 +76,18 @@ static TopicPathBuilder newBuilder(final PolicyId policyId) { return ProtocolFactory.newTopicPathBuilder(policyId); } + /** + * Returns a mutable builder to create immutable {@code TopicPath} instances for a given {@code connectionId}. + * + * @param connectionId the identifier of the {@code Connection}. + * @return the builder. + * @throws NullPointerException if {@code connectionId} is {@code null}. + * @since 3.1.0 + */ + static TopicPathBuilder newBuilder(final ConnectionId connectionId) { + return ProtocolFactory.newTopicPathBuilder(connectionId); + } + /** * Returns a mutable builder to create immutable {@code TopicPath} instances for a given {@code namespace}. * @@ -122,12 +142,20 @@ static TopicPathBuilder fromNamespace(final String namespace) { Optional getAction(); /** - * Returns an {@link Optional} for an search action part of this {@code TopicPath}. + * Returns an {@link Optional} for a search action part of this {@code TopicPath}. * * @return the search action. */ Optional getSearchAction(); + /** + * Returns an {@link Optional} for a streaming action part of this {@code TopicPath}. + * + * @return the streaming action. + * @since 3.2.0 + */ + Optional getStreamingAction(); + /** * Returns an {@link Optional} for a subject part of this {@code TopicPath}. * @@ -268,7 +296,14 @@ enum Criterion { * * @since 2.0.0 */ - ANNOUNCEMENTS("announcements"); + ANNOUNCEMENTS("announcements"), + + /** + * Criterion for streaming commands. + * + * @since 3.2.0 + */ + STREAMING("streaming"); private final String name; @@ -482,4 +517,65 @@ public String toString() { } + /** + * An enumeration of topic path streaming actions. + * + * @since 3.2.0 + */ + enum StreamingAction { + + SUBSCRIBE_FOR_PERSISTED_EVENTS(SubscribeForPersistedEvents.NAME), + + CANCEL(CancelStreamingSubscription.NAME), + + REQUEST(RequestFromStreamingSubscription.NAME), + + COMPLETE(StreamingSubscriptionComplete.NAME), + + GENERATED(StreamingSubscriptionCreated.NAME), + + FAILED(StreamingSubscriptionFailed.NAME), + + NEXT(StreamingSubscriptionHasNext.NAME), + + ERROR("error"); + + private final String name; + + StreamingAction(final String name) { + this.name = name; + } + + /** + * Creates a StreamingAction from the passed StreamingAction {@code name} if such an enum value exists, + * otherwise an empty Optional. + * + * @param name the StreamingAction name to create the StreamingAction enum value of. + * @return the optional StreamingAction. + */ + public static Optional forName(final String name) { + return Stream.of(values()) + .filter(a -> Objects.equals(a.getName(), name)) + .findFirst(); + } + + /** + * Returns the StreamingAction name as String. + * + * @return the StreamingAction name as String. + */ + public String getName() { + return name; + } + + /** + * @return the same as {@link #getName()}. + */ + @Override + public String toString() { + return getName(); + } + + } + } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPathBuilder.java b/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPathBuilder.java index 6c4f9db8e5e..31532183d0a 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPathBuilder.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/TopicPathBuilder.java @@ -64,6 +64,15 @@ public interface TopicPathBuilder { */ AnnouncementsTopicPathBuilder announcements(); + /** + * Sets the {@code Group} of this builder to {@link TopicPath.Criterion#STREAMING}. A previously set group is + * replaced. + * + * @return this builder. + * @since 3.2.0 + */ + StreamingTopicPathBuilder streaming(); + /** * Sets the {@code Channel} of this builder to {@link TopicPath.Channel#TWIN}. A previously set channel is * replaced. diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AbstractStreamingMessageAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AbstractStreamingMessageAdapter.java new file mode 100644 index 00000000000..161eb40311d --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AbstractStreamingMessageAdapter.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.adapter; + +import java.util.EnumSet; +import java.util.Set; + +import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.mapper.SignalMapper; +import org.eclipse.ditto.protocol.mappingstrategies.MappingStrategies; + +/** + * Adapter for mapping a "streaming" {@link Signal} to and from an {@link Adaptable}. + * + * @param the type of the signals mapped by this adapter. + */ +abstract class AbstractStreamingMessageAdapter> extends AbstractAdapter + implements Adapter { + + private final SignalMapper signalMapper; + + AbstractStreamingMessageAdapter( + final MappingStrategies mappingStrategies, + final SignalMapper signalMapper, + final HeaderTranslator headerTranslator) { + + super(mappingStrategies, headerTranslator, EmptyPathMatcher.getInstance()); + this.signalMapper = signalMapper; + } + + @Override + protected Adaptable mapSignalToAdaptable(final T signal, final TopicPath.Channel channel) { + return signalMapper.mapSignalToAdaptable(signal, channel); + } + + @Override + public Adaptable toAdaptable(final T t) { + return toAdaptable(t, TopicPath.Channel.LIVE); + } + + @Override + public TopicPath toTopicPath(final T signal, final TopicPath.Channel channel) { + return signalMapper.mapSignalToTopicPath(signal, channel); + } + + @Override + public Set getGroups() { + return EnumSet.of(TopicPath.Group.POLICIES, TopicPath.Group.THINGS, TopicPath.Group.CONNECTIONS); + } + + @Override + public Set getChannels() { + return EnumSet.of(TopicPath.Channel.NONE, TopicPath.Channel.TWIN); + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/Adapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/Adapter.java index 8a8ec6b22b8..4bd0a3a024e 100755 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/Adapter.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/Adapter.java @@ -113,6 +113,17 @@ default Set getSearchActions() { return Collections.emptySet(); } + /** + * Return the set of streaming actions supported by this adapter. + * It is the empty set by default. + * + * @return the collection of supported streaming actions. + * @since 3.2.0 + */ + default Set getStreamingActions() { + return Collections.emptySet(); + } + /** * Retrieve whether this adapter is for responses. * diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AdapterResolverBySignal.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AdapterResolverBySignal.java index ed3bdfdb389..6e8ca806a45 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AdapterResolverBySignal.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/AdapterResolverBySignal.java @@ -23,7 +23,9 @@ import org.eclipse.ditto.base.model.signals.acks.Acknowledgements; import org.eclipse.ditto.base.model.signals.commands.Command; import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; import org.eclipse.ditto.base.model.signals.events.Event; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.connectivity.model.signals.announcements.ConnectivityAnnouncement; import org.eclipse.ditto.messages.model.signals.commands.MessageCommand; import org.eclipse.ditto.messages.model.signals.commands.MessageCommandResponse; @@ -33,6 +35,7 @@ import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommand; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.protocol.TopicPath; import org.eclipse.ditto.protocol.UnknownChannelException; import org.eclipse.ditto.protocol.UnknownSignalException; @@ -61,16 +64,22 @@ final class AdapterResolverBySignal { private final PolicyCommandAdapterProvider policiesAdapters; private final ConnectivityCommandAdapterProvider connectivityAdapters; private final AcknowledgementAdapterProvider acknowledgementAdapters; + private final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter; + private final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter; AdapterResolverBySignal(final ThingCommandAdapterProvider thingsAdapters, final PolicyCommandAdapterProvider policiesAdapters, final ConnectivityCommandAdapterProvider connectivityAdapters, - final AcknowledgementAdapterProvider acknowledgementAdapters) { + final AcknowledgementAdapterProvider acknowledgementAdapters, + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter, + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter) { this.thingsAdapters = thingsAdapters; this.policiesAdapters = policiesAdapters; this.connectivityAdapters = connectivityAdapters; this.acknowledgementAdapters = acknowledgementAdapters; + this.streamingSubscriptionCommandAdapter = streamingSubscriptionCommandAdapter; + this.streamingSubscriptionEventAdapter = streamingSubscriptionEventAdapter; } @SuppressWarnings("unchecked") @@ -110,10 +119,19 @@ private > Adapter resolveEvent(final Event event, fina validateChannel(channel, event, LIVE, TWIN); return (Adapter) thingsAdapters.getEventAdapter(); } + if (event instanceof PolicyEvent) { + validateChannel(channel, event, NONE); + return (Adapter) policiesAdapters.getEventAdapter(); + } + if (event instanceof SubscriptionEvent) { validateNotLive(event); return (Adapter) thingsAdapters.getSubscriptionEventAdapter(); } + if (event instanceof StreamingSubscriptionEvent) { + validateNotLive(event); + return (Adapter) streamingSubscriptionEventAdapter; + } throw UnknownSignalException.newBuilder(event.getName()) .dittoHeaders(event.getDittoHeaders()) @@ -212,6 +230,10 @@ private > Adapter resolveCommand(final Command command validateNotLive(command); return (Adapter) thingsAdapters.getSearchCommandAdapter(); } + if (command instanceof StreamingSubscriptionCommand) { + validateNotLive(command); + return (Adapter) streamingSubscriptionCommandAdapter; + } if (command instanceof PolicyModifyCommand) { validateChannel(channel, command, NONE); diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DefaultAdapterResolver.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DefaultAdapterResolver.java index fd7b522b4d0..44717e7fcfb 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DefaultAdapterResolver.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DefaultAdapterResolver.java @@ -43,15 +43,19 @@ final class DefaultAdapterResolver implements AdapterResolver { DefaultAdapterResolver(final ThingCommandAdapterProvider thingsAdapters, final PolicyCommandAdapterProvider policiesAdapters, final ConnectivityCommandAdapterProvider connectivityAdapters, - final AcknowledgementAdapterProvider acknowledgementAdapters) { + final AcknowledgementAdapterProvider acknowledgementAdapters, + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter, + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter) { final List> adapters = new ArrayList<>(); adapters.addAll(thingsAdapters.getAdapters()); adapters.addAll(policiesAdapters.getAdapters()); adapters.addAll(connectivityAdapters.getAdapters()); adapters.addAll(acknowledgementAdapters.getAdapters()); + adapters.add(streamingSubscriptionCommandAdapter); + adapters.add(streamingSubscriptionEventAdapter); resolver = computeResolver(adapters); resolverBySignal = new AdapterResolverBySignal(thingsAdapters, policiesAdapters, connectivityAdapters, - acknowledgementAdapters); + acknowledgementAdapters, streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter); } @Override @@ -236,6 +240,8 @@ private static Function> computeResolver(final List(TopicPath.SearchAction.class, TopicPath.SearchAction.values(), Adapter::getSearchActions, forTopicPath(TopicPath::getSearchAction)), + new ForEnumOptional<>(TopicPath.StreamingAction.class, TopicPath.StreamingAction.values(), + Adapter::getStreamingActions, forTopicPath(TopicPath::getStreamingAction)), new ForEnum<>(Bool.class, Bool.values(), Bool.composeAsSet(Adapter::isForResponses), Bool.compose(DefaultAdapterResolver::isResponse)), new ForEnum<>(Bool.class, Bool.values(), Bool.composeAsSet(Adapter::requiresSubject), diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapter.java index 3a22c9f4b1f..4f2fa4993bb 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapter.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapter.java @@ -43,6 +43,9 @@ public final class DittoProtocolAdapter implements ProtocolAdapter { private final PolicyCommandAdapterProvider policiesAdapters; private final ConnectivityCommandAdapterProvider connectivityAdapters; private final AcknowledgementAdapterProvider acknowledgementAdapters; + + private final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter; + private final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter; private final AdapterResolver adapterResolver; private DittoProtocolAdapter(final ErrorRegistry errorRegistry, @@ -52,8 +55,10 @@ private DittoProtocolAdapter(final ErrorRegistry errorReg this.policiesAdapters = new DefaultPolicyCommandAdapterProvider(errorRegistry, headerTranslator); this.connectivityAdapters = new DefaultConnectivityCommandAdapterProvider(headerTranslator); this.acknowledgementAdapters = new DefaultAcknowledgementsAdapterProvider(errorRegistry, headerTranslator); + streamingSubscriptionCommandAdapter = StreamingSubscriptionCommandAdapter.of(headerTranslator); + streamingSubscriptionEventAdapter = StreamingSubscriptionEventAdapter.of(headerTranslator, errorRegistry); this.adapterResolver = new DefaultAdapterResolver(thingsAdapters, policiesAdapters, connectivityAdapters, - acknowledgementAdapters); + acknowledgementAdapters, streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter); } private DittoProtocolAdapter(final HeaderTranslator headerTranslator, @@ -61,12 +66,18 @@ private DittoProtocolAdapter(final HeaderTranslator headerTranslator, final PolicyCommandAdapterProvider policiesAdapters, final ConnectivityCommandAdapterProvider connectivityAdapters, final AcknowledgementAdapterProvider acknowledgementAdapters, + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter, + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter, final AdapterResolver adapterResolver) { this.headerTranslator = checkNotNull(headerTranslator, "headerTranslator"); this.thingsAdapters = checkNotNull(thingsAdapters, "thingsAdapters"); this.policiesAdapters = checkNotNull(policiesAdapters, "policiesAdapters"); this.connectivityAdapters = checkNotNull(connectivityAdapters, "connectivityAdapters"); this.acknowledgementAdapters = checkNotNull(acknowledgementAdapters, "acknowledgementAdapters"); + this.streamingSubscriptionCommandAdapter = checkNotNull(streamingSubscriptionCommandAdapter, + "streamingSubscriptionCommandAdapter"); + this.streamingSubscriptionEventAdapter = checkNotNull(streamingSubscriptionEventAdapter, + "streamingSubscriptionEventAdapter"); this.adapterResolver = checkNotNull(adapterResolver, "adapterResolver"); } @@ -106,6 +117,8 @@ public static HeaderTranslator getHeaderTranslator() { * @param policyCommandAdapterProvider command adapters for policy commands * @param connectivityAdapters adapters for connectivity commands. * @param acknowledgementAdapters adapters for acknowledgements. + * @param streamingSubscriptionCommandAdapter adapters for streaming subscription commands. + * @param streamingSubscriptionEventAdapter adapters for streaming subscription events. * @param adapterResolver resolves the correct adapter from a command * @return new instance of {@link DittoProtocolAdapter} */ @@ -114,9 +127,12 @@ static DittoProtocolAdapter newInstance(final HeaderTranslator headerTranslator, final PolicyCommandAdapterProvider policyCommandAdapterProvider, final ConnectivityCommandAdapterProvider connectivityAdapters, final AcknowledgementAdapterProvider acknowledgementAdapters, + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter, + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter, final AdapterResolver adapterResolver) { return new DittoProtocolAdapter(headerTranslator, thingCommandAdapterProvider, policyCommandAdapterProvider, - connectivityAdapters, acknowledgementAdapters, adapterResolver + connectivityAdapters, acknowledgementAdapters, + streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter, adapterResolver ); } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/ProtocolAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/ProtocolAdapter.java index 05cd69b599f..ebd67556eab 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/ProtocolAdapter.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/ProtocolAdapter.java @@ -19,10 +19,14 @@ import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.base.model.signals.announcements.Announcement; +import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; +import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommandResponse; +import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.messages.model.signals.commands.MessageCommand; import org.eclipse.ditto.messages.model.signals.commands.MessageCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.PolicyCommand; import org.eclipse.ditto.policies.model.signals.commands.PolicyCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.protocol.Adaptable; import org.eclipse.ditto.protocol.TopicPath; @@ -112,7 +116,11 @@ static TopicPath.Channel determineChannel(final Signal signal) { * @return the default channel determined from the signal */ static TopicPath.Channel determineDefaultChannel(final Signal signal) { - if (signal instanceof PolicyCommand || signal instanceof PolicyCommandResponse) { + if (signal instanceof PolicyCommand || signal instanceof PolicyCommandResponse || + signal instanceof PolicyEvent) { + return NONE; + } else if (signal instanceof ConnectivityCommand || signal instanceof ConnectivityCommandResponse || + signal instanceof ConnectivityEvent) { return NONE; } else if (signal instanceof Announcement) { return NONE; diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionCommandAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionCommandAdapter.java new file mode 100644 index 00000000000..a7a626a6e71 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionCommandAdapter.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.adapter; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.mapper.SignalMapperFactory; +import org.eclipse.ditto.protocol.mappingstrategies.MappingStrategiesFactory; + +/** + * Adapter for mapping a {@link StreamingSubscriptionCommand} to and from an {@link Adaptable}. + * + * @since 3.2.0 + */ +public final class StreamingSubscriptionCommandAdapter + extends AbstractStreamingMessageAdapter> { + + private StreamingSubscriptionCommandAdapter(final HeaderTranslator headerTranslator) { + super(MappingStrategiesFactory.getStreamingSubscriptionCommandMappingStrategies(), + SignalMapperFactory.newStreamingSubscriptionCommandSignalMapper(), + headerTranslator + ); + } + + /** + * Returns a new StreamingSubscriptionCommandAdapter. + * + * @param headerTranslator translator between external and Ditto headers. + * @return the adapter. + */ + public static StreamingSubscriptionCommandAdapter of(final HeaderTranslator headerTranslator) { + return new StreamingSubscriptionCommandAdapter(checkNotNull(headerTranslator, "headerTranslator")); + } + + @Override + protected String getType(final Adaptable adaptable) { + return StreamingSubscriptionCommand.TYPE_PREFIX + adaptable.getTopicPath().getStreamingAction().orElse(null); + } + + @Override + public Set getCriteria() { + return EnumSet.of(TopicPath.Criterion.STREAMING); + } + + @Override + public Set getActions() { + return Collections.emptySet(); + } + + @Override + public boolean isForResponses() { + return false; + } + + @Override + public Set getStreamingActions() { + return EnumSet.of(TopicPath.StreamingAction.SUBSCRIBE_FOR_PERSISTED_EVENTS, TopicPath.StreamingAction.REQUEST, + TopicPath.StreamingAction.CANCEL); + } +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionEventAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionEventAdapter.java new file mode 100644 index 00000000000..f6f22735793 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/StreamingSubscriptionEventAdapter.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.adapter; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.base.model.signals.ErrorRegistry; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.mapper.SignalMapperFactory; +import org.eclipse.ditto.protocol.mappingstrategies.MappingStrategiesFactory; + +/** + * Adapter for mapping a {@link StreamingSubscriptionEvent} to and from an {@link Adaptable}. + * + * @since 3.2.0 + */ +public final class StreamingSubscriptionEventAdapter + extends AbstractStreamingMessageAdapter> { + + private StreamingSubscriptionEventAdapter(final HeaderTranslator headerTranslator, + final ErrorRegistry errorRegistry) { + super(MappingStrategiesFactory.getStreamingSubscriptionEventMappingStrategies(errorRegistry), + SignalMapperFactory.newStreamingSubscriptionEventSignalMapper(), + headerTranslator + ); + } + + /** + * Returns a new StreamingSubscriptionEventAdapter. + * + * @param headerTranslator translator between external and Ditto headers. + * @param errorRegistry the error registry for {@code SubscriptionFailed} events. + * @return the adapter. + */ + public static StreamingSubscriptionEventAdapter of(final HeaderTranslator headerTranslator, + final ErrorRegistry errorRegistry) { + return new StreamingSubscriptionEventAdapter(checkNotNull(headerTranslator, "headerTranslator"), + checkNotNull(errorRegistry, "errorRegistry")); + } + + @Override + protected String getType(final Adaptable adaptable) { + return StreamingSubscriptionEvent.TYPE_PREFIX + adaptable.getTopicPath().getStreamingAction().orElse(null); + } + + @Override + public Set getCriteria() { + return EnumSet.of(TopicPath.Criterion.STREAMING); + } + + @Override + public Set getActions() { + return Collections.emptySet(); + } + + @Override + public boolean isForResponses() { + return false; + } + + @Override + public Set getStreamingActions() { + return EnumSet.of(TopicPath.StreamingAction.COMPLETE, TopicPath.StreamingAction.NEXT, + TopicPath.StreamingAction.FAILED, TopicPath.StreamingAction.GENERATED); + } + + @Override + public boolean supportsWildcardTopics() { + return false; + } +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/connectivity/package-info.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/connectivity/package-info.java new file mode 100755 index 00000000000..51b170fb9b5 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/connectivity/package-info.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.eclipse.ditto.utils.jsr305.annotations.AllValuesAreNonnullByDefault +package org.eclipse.ditto.protocol.adapter.connectivity; diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/DefaultPolicyCommandAdapterProvider.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/DefaultPolicyCommandAdapterProvider.java index 37425b6bd07..1fe1bbfd221 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/DefaultPolicyCommandAdapterProvider.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/DefaultPolicyCommandAdapterProvider.java @@ -24,6 +24,7 @@ import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommand; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.protocol.adapter.Adapter; import org.eclipse.ditto.protocol.adapter.provider.PolicyCommandAdapterProvider; @@ -40,6 +41,7 @@ public final class DefaultPolicyCommandAdapterProvider implements PolicyCommandA private final PolicyModifyCommandResponseAdapter policyModifyCommandResponseAdapter; private final PolicyQueryCommandResponseAdapter policyQueryCommandResponseAdapter; private final PolicyAnnouncementAdapter policyAnnouncementAdapter; + private final PolicyEventAdapter policyEventAdapter; public DefaultPolicyCommandAdapterProvider(final ErrorRegistry errorRegistry, final HeaderTranslator headerTranslator) { @@ -49,6 +51,7 @@ public DefaultPolicyCommandAdapterProvider(final ErrorRegistry getErrorResponseAdapter() { @@ -75,6 +78,11 @@ public Adapter> getAnnouncementAdapter() { return policyAnnouncementAdapter; } + @Override + public Adapter> getEventAdapter() { + return policyEventAdapter; + } + @Override public List> getAdapters() { return Arrays.asList( @@ -83,7 +91,8 @@ public List> getAdapters() { policyQueryCommandAdapter, policyModifyCommandResponseAdapter, policyQueryCommandResponseAdapter, - policyAnnouncementAdapter + policyAnnouncementAdapter, + policyEventAdapter ); } } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/PolicyEventAdapter.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/PolicyEventAdapter.java new file mode 100644 index 00000000000..589a9a1899c --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/policies/PolicyEventAdapter.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.adapter.policies; + +import static java.util.Objects.requireNonNull; + +import org.eclipse.ditto.base.model.headers.translator.HeaderTranslator; +import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.adapter.AbstractAdapter; +import org.eclipse.ditto.protocol.adapter.EventAdapter; +import org.eclipse.ditto.protocol.mapper.SignalMapperFactory; +import org.eclipse.ditto.protocol.mappingstrategies.MappingStrategiesFactory; + +/** + * Adapter for mapping a {@link PolicyEvent} to and from an {@link org.eclipse.ditto.protocol.Adaptable}. + */ +final class PolicyEventAdapter extends AbstractPolicyAdapter> implements EventAdapter> { + + private PolicyEventAdapter(final HeaderTranslator headerTranslator) { + super(MappingStrategiesFactory.getPolicyEventMappingStrategies(), + SignalMapperFactory.newPolicyEventSignalMapper(), + headerTranslator); + } + + /** + * Returns a new PolicyEventAdapter. + * + * @param headerTranslator translator between external and Ditto headers. + * @return the adapter. + */ + public static PolicyEventAdapter of(final HeaderTranslator headerTranslator) { + return new PolicyEventAdapter(requireNonNull(headerTranslator)); + } + + private static String getActionNameWithFirstLetterUpperCase(final TopicPath topicPath) { + return topicPath.getAction() + .map(TopicPath.Action::toString) + .map(AbstractAdapter::upperCaseFirst) + .orElseThrow(() -> new NullPointerException("TopicPath did not contain an Action!")); + } + + @Override + protected String getType(final Adaptable adaptable) { + final TopicPath topicPath = adaptable.getTopicPath(); + final JsonPointer path = adaptable.getPayload().getPath(); + final String eventName = payloadPathMatcher.match(path) + getActionNameWithFirstLetterUpperCase(topicPath); + return topicPath.getGroup() + "." + topicPath.getCriterion() + ":" + eventName; + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/provider/PolicyCommandAdapterProvider.java b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/provider/PolicyCommandAdapterProvider.java index 080b6910fa3..09daf183b45 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/provider/PolicyCommandAdapterProvider.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/adapter/provider/PolicyCommandAdapterProvider.java @@ -12,13 +12,14 @@ */ package org.eclipse.ditto.protocol.adapter.provider; -import org.eclipse.ditto.protocol.adapter.Adapter; import org.eclipse.ditto.policies.model.signals.announcements.PolicyAnnouncement; import org.eclipse.ditto.policies.model.signals.commands.PolicyErrorResponse; import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommand; import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommand; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.protocol.adapter.Adapter; /** * Provider for all policy command adapters. This interface mainly defines the generic type arguments and adds @@ -30,6 +31,7 @@ public interface PolicyCommandAdapterProvider extends QueryCommandAdapterProvider, PolicyQueryCommandResponse>, ModifyCommandAdapterProvider, PolicyModifyCommandResponse>, ErrorResponseAdapterProvider, + EventAdapterProvider>, AdapterProvider { /** diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/AbstractCommandSignalMapper.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/AbstractCommandSignalMapper.java index a6fce7e9d97..10182d5902e 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/AbstractCommandSignalMapper.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/AbstractCommandSignalMapper.java @@ -15,12 +15,12 @@ import java.util.stream.Stream; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.signals.Signal; import org.eclipse.ditto.protocol.CommandsTopicPathBuilder; import org.eclipse.ditto.protocol.TopicPath; import org.eclipse.ditto.protocol.TopicPathBuilder; import org.eclipse.ditto.protocol.UnknownCommandException; import org.eclipse.ditto.protocol.UnknownCommandResponseException; -import org.eclipse.ditto.base.model.signals.Signal; /** * Base class of {@link SignalMapper}s for commands (e.g. query, modify commands). @@ -45,7 +45,7 @@ TopicPath getTopicPath(final T command, final TopicPath.Channel channel) { abstract TopicPathBuilder getTopicPathBuilder(final T command); /** - * @return array of {@link org.eclipse.ditto.protocol.TopicPath.Action}s the implementation supports. + * @return array of {@link TopicPath.Action}s the implementation supports. */ abstract TopicPath.Action[] getSupportedActions(); diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/PolicyEventSignalMapper.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/PolicyEventSignalMapper.java new file mode 100644 index 00000000000..343d730dd52 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/PolicyEventSignalMapper.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mapper; + +import java.util.Locale; +import java.util.Optional; + +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.json.JsonValue; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.protocol.EventsTopicPathBuilder; +import org.eclipse.ditto.protocol.PayloadBuilder; +import org.eclipse.ditto.protocol.ProtocolFactory; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.TopicPathBuilder; +import org.eclipse.ditto.protocol.UnknownChannelException; +import org.eclipse.ditto.protocol.UnknownEventException; + +final class PolicyEventSignalMapper extends AbstractSignalMapper> { + + @Override + void enhancePayloadBuilder(final PolicyEvent signal, final PayloadBuilder payloadBuilder) { + payloadBuilder.withRevision(signal.getRevision()) + .withTimestamp(signal.getTimestamp().orElse(null)); + final Optional value = + signal.getEntity(signal.getDittoHeaders().getSchemaVersion().orElse(signal.getLatestSchemaVersion())); + value.ifPresent(payloadBuilder::withValue); + } + + @Override + DittoHeaders enhanceHeaders(final PolicyEvent signal) { + final Optional value = + signal.getEntity(signal.getDittoHeaders().getSchemaVersion().orElse(signal.getLatestSchemaVersion())); + if (value.isPresent()) { + return ProtocolFactory.newHeadersWithJsonContentType(signal.getDittoHeaders()); + } else { + return signal.getDittoHeaders(); + } + } + + @Override + TopicPath getTopicPath(final PolicyEvent signal, final TopicPath.Channel channel) { + final EventsTopicPathBuilder topicPathBuilder = getEventsTopicPathBuilderOrThrow(signal, channel); + final String eventName = getLowerCaseEventName(signal); + if (isAction(eventName, TopicPath.Action.CREATED)) { + topicPathBuilder.created(); + } else if (isAction(eventName, TopicPath.Action.MODIFIED)) { + topicPathBuilder.modified(); + } else if (isAction(eventName, TopicPath.Action.DELETED)) { + topicPathBuilder.deleted(); + } else if (isAction(eventName, TopicPath.Action.MERGED)) { + topicPathBuilder.merged(); + } else { + throw UnknownEventException.newBuilder(eventName).build(); + } + return topicPathBuilder.build(); + } + + private static EventsTopicPathBuilder getEventsTopicPathBuilderOrThrow(final PolicyEvent event, + final TopicPath.Channel channel) { + + TopicPathBuilder topicPathBuilder = ProtocolFactory.newTopicPathBuilder(event.getEntityId()); + if (TopicPath.Channel.NONE == channel) { + topicPathBuilder = topicPathBuilder.none(); + } else { + throw UnknownChannelException.newBuilder(channel, event.getType()) + .dittoHeaders(event.getDittoHeaders()) + .build(); + } + return topicPathBuilder.events(); + } + + private static String getLowerCaseEventName(final PolicyEvent thingEvent) { + final Class thingEventClass = thingEvent.getClass(); + final String eventClassSimpleName = thingEventClass.getSimpleName(); + return eventClassSimpleName.toLowerCase(Locale.ENGLISH); + } + + private static boolean isAction(final String eventName, final TopicPath.Action expectedAction) { + return eventName.contains(expectedAction.getName()); + } +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/SignalMapperFactory.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/SignalMapperFactory.java index 81ad891cd9e..40f25b25f2c 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/SignalMapperFactory.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/SignalMapperFactory.java @@ -12,6 +12,8 @@ */ package org.eclipse.ditto.protocol.mapper; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; import org.eclipse.ditto.connectivity.model.signals.announcements.ConnectivityAnnouncement; import org.eclipse.ditto.messages.model.signals.commands.MessageCommand; import org.eclipse.ditto.messages.model.signals.commands.MessageCommandResponse; @@ -20,6 +22,7 @@ import org.eclipse.ditto.policies.model.signals.commands.modify.PolicyModifyCommandResponse; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommand; import org.eclipse.ditto.policies.model.signals.commands.query.PolicyQueryCommandResponse; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; import org.eclipse.ditto.things.model.signals.commands.modify.MergeThing; import org.eclipse.ditto.things.model.signals.commands.modify.MergeThingResponse; import org.eclipse.ditto.things.model.signals.commands.modify.ThingModifyCommand; @@ -80,6 +83,14 @@ public static SignalMapper> newSubscriptionEventSignalMappe return new SubscriptionEventSignalMapper(); } + public static SignalMapper> newStreamingSubscriptionCommandSignalMapper() { + return new StreamingSubscriptionCommandSignalMapper<>(); + } + + public static SignalMapper> newStreamingSubscriptionEventSignalMapper() { + return new StreamingSubscriptionEventSignalMapper(); + } + public static SignalMapper newRetrieveThingsSignalMapper() { return new RetrieveThingsSignalMapper(); } @@ -112,6 +123,10 @@ public static SignalMapper> newPolicyAnnouncementSignalMap return new PolicyAnnouncementSignalMapper(); } + public static SignalMapper> newPolicyEventSignalMapper() { + return new PolicyEventSignalMapper(); + } + public static SignalMapper> newMessageCommandSignalMapper() { return MessageSignalMapper.getInstance(); } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionCommandSignalMapper.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionCommandSignalMapper.java new file mode 100644 index 00000000000..49e1e1b6e57 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionCommandSignalMapper.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mapper; + +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.protocol.PayloadBuilder; +import org.eclipse.ditto.protocol.ProtocolFactory; +import org.eclipse.ditto.protocol.StreamingTopicPathBuilder; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.TopicPathBuilder; +import org.eclipse.ditto.protocol.UnknownCommandException; + +/** + * Signal mapper implementation for {@link StreamingSubscriptionCommand}s. + * + * @param the type of the command + */ +final class StreamingSubscriptionCommandSignalMapper> + extends AbstractSignalMapper { + + @Override + TopicPath getTopicPath(final T command, final TopicPath.Channel channel) { + final TopicPathBuilder topicPathBuilder = getTopicPathBuilder(command); + final StreamingTopicPathBuilder streamingTopicPathBuilder = + fromTopicPathBuilderWithChannel(topicPathBuilder, channel); + setTopicPathAction(streamingTopicPathBuilder, command, getSupportedActions()); + return streamingTopicPathBuilder.build(); + } + + /** + * @return array of {@link org.eclipse.ditto.protocol.TopicPath.Action}s the implementation supports. + */ + public TopicPath.StreamingAction[] getSupportedActions() { + return new TopicPath.StreamingAction[]{ + TopicPath.StreamingAction.REQUEST, + TopicPath.StreamingAction.CANCEL, + TopicPath.StreamingAction.SUBSCRIBE_FOR_PERSISTED_EVENTS + }; + } + + @Override + void enhancePayloadBuilder(final T command, final PayloadBuilder payloadBuilder) { + + final JsonObjectBuilder payloadContentBuilder = JsonFactory.newObjectBuilder(); + if (command instanceof SubscribeForPersistedEvents) { + final SubscribeForPersistedEvents subscribeCommand = (SubscribeForPersistedEvents) command; + payloadContentBuilder + .set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_REVISION, + subscribeCommand.getFromHistoricalRevision()) + .set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_REVISION, + subscribeCommand.getToHistoricalRevision()); + subscribeCommand.getFromHistoricalTimestamp().ifPresent(fromTs -> + payloadContentBuilder.set(SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP, + fromTs.toString())); + subscribeCommand.getToHistoricalTimestamp().ifPresent(toTs -> + payloadContentBuilder.set(SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_TIMESTAMP, + toTs.toString())); + } else if (command instanceof CancelStreamingSubscription) { + final CancelStreamingSubscription cancelCommand = (CancelStreamingSubscription) command; + payloadContentBuilder + .set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, cancelCommand.getSubscriptionId()); + } else if (command instanceof RequestFromStreamingSubscription) { + final RequestFromStreamingSubscription requestCommand = (RequestFromStreamingSubscription) command; + payloadContentBuilder + .set(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID, requestCommand.getSubscriptionId()) + .set(RequestFromStreamingSubscription.JsonFields.DEMAND, requestCommand.getDemand()); + } else { + throw UnknownCommandException.newBuilder(command.getClass().toString()).build(); + } + payloadBuilder.withValue(payloadContentBuilder.build()); + } + + private static StreamingTopicPathBuilder fromTopicPathBuilderWithChannel(final TopicPathBuilder topicPathBuilder, + final TopicPath.Channel channel) { + + if (channel == TopicPath.Channel.TWIN) { + return topicPathBuilder.twin().streaming(); + } else if (channel == TopicPath.Channel.NONE) { + return topicPathBuilder.none().streaming(); + } else { + throw new IllegalArgumentException("Unknown or unsupported Channel '" + channel + "'"); + } + } + + private TopicPathBuilder getTopicPathBuilder(final StreamingSubscriptionCommand command) { + return ProtocolFactory.newTopicPathBuilder(command.getEntityId()); + } + + private void setTopicPathAction(final StreamingTopicPathBuilder builder, final T command, + final TopicPath.StreamingAction... supportedActions) { + + if (supportedActions.length > 0) { + final String streamingCommandName = command.getName(); + final TopicPath.StreamingAction streamingAction = + TopicPath.StreamingAction.forName(streamingCommandName) + .orElseThrow(() -> unknownCommandException(streamingCommandName)); + setAction(builder, streamingAction); + } + } + + DittoRuntimeException unknownCommandException(final String commandName) { + return UnknownCommandException.newBuilder(commandName).build(); + } + + private void setAction(final StreamingTopicPathBuilder builder, final TopicPath.StreamingAction streamingAction) { + switch (streamingAction) { + case SUBSCRIBE_FOR_PERSISTED_EVENTS: + builder.subscribe(SubscribeForPersistedEvents.NAME); + break; + case REQUEST: + builder.request(); + break; + case CANCEL: + builder.cancel(); + break; + default: + throw unknownCommandException(streamingAction.getName()); + } + } +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionEventSignalMapper.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionEventSignalMapper.java new file mode 100644 index 00000000000..65755188cdf --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mapper/StreamingSubscriptionEventSignalMapper.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mapper; + +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionCreated; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionHasNext; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.policies.model.PolicyConstants; +import org.eclipse.ditto.policies.model.PolicyId; +import org.eclipse.ditto.protocol.PayloadBuilder; +import org.eclipse.ditto.protocol.ProtocolFactory; +import org.eclipse.ditto.protocol.TopicPath; +import org.eclipse.ditto.protocol.TopicPathBuilder; +import org.eclipse.ditto.protocol.UnknownEventException; +import org.eclipse.ditto.things.model.ThingConstants; +import org.eclipse.ditto.things.model.ThingId; + +/** + * Signal mapper implementation for {@link StreamingSubscriptionEvent}s. + */ +final class StreamingSubscriptionEventSignalMapper extends AbstractSignalMapper> { + + private static final JsonFieldDefinition SUBSCRIPTION_ID = + WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID; + + @Override + void enhancePayloadBuilder(final StreamingSubscriptionEvent signal, final PayloadBuilder payloadBuilder) { + final JsonObjectBuilder payloadContentBuilder = JsonFactory.newObjectBuilder(); + if (signal instanceof StreamingSubscriptionCreated) { + final StreamingSubscriptionCreated createdEvent = (StreamingSubscriptionCreated) signal; + payloadContentBuilder.set(SUBSCRIPTION_ID, createdEvent.getSubscriptionId()); + + } else if (signal instanceof StreamingSubscriptionComplete) { + final StreamingSubscriptionComplete completedEvent = (StreamingSubscriptionComplete) signal; + payloadContentBuilder.set(SUBSCRIPTION_ID, completedEvent.getSubscriptionId()); + + } else if (signal instanceof StreamingSubscriptionFailed) { + final StreamingSubscriptionFailed failedEvent = (StreamingSubscriptionFailed) signal; + payloadContentBuilder + .set(SUBSCRIPTION_ID, failedEvent.getSubscriptionId()) + .set(StreamingSubscriptionFailed.JsonFields.ERROR, failedEvent.getError().toJson()); + + } else if (signal instanceof StreamingSubscriptionHasNext) { + final StreamingSubscriptionHasNext hasNext = (StreamingSubscriptionHasNext) signal; + payloadContentBuilder + .set(SUBSCRIPTION_ID, hasNext.getSubscriptionId()) + .set(StreamingSubscriptionHasNext.JsonFields.ITEM, hasNext.getItem()); + + } else { + throw UnknownEventException.newBuilder(signal.getClass().getCanonicalName()).build(); + } + payloadBuilder.withValue(payloadContentBuilder.build()); + } + + @Override + DittoHeaders enhanceHeaders(final StreamingSubscriptionEvent signal) { + return ProtocolFactory.newHeadersWithJsonContentType(signal.getDittoHeaders()); + } + + @Override + TopicPath getTopicPath(final StreamingSubscriptionEvent signal, final TopicPath.Channel channel) { + + final TopicPathBuilder topicPathBuilder; + if (signal.getEntityType().equals(ThingConstants.ENTITY_TYPE)) { + topicPathBuilder = TopicPath.newBuilder(ThingId.of(signal.getEntityId())).things().twin(); + } else if (signal.getEntityType().equals(PolicyConstants.ENTITY_TYPE)) { + topicPathBuilder = TopicPath.newBuilder(PolicyId.of(signal.getEntityId())).policies().none(); + } else if (signal.getEntityType().equals(ConnectivityConstants.ENTITY_TYPE)) { + topicPathBuilder = TopicPath.newBuilder(ConnectionId.of(signal.getEntityId())).connections().none(); + } else { + throw UnknownEventException.newBuilder(signal.getClass().getCanonicalName()).build(); + } + + final TopicPath topicPath; + if (signal instanceof StreamingSubscriptionCreated) { + topicPath = topicPathBuilder.streaming().generated().build(); + } else if (signal instanceof StreamingSubscriptionComplete) { + topicPath = topicPathBuilder.streaming().complete().build(); + } else if (signal instanceof StreamingSubscriptionFailed) { + topicPath = topicPathBuilder.streaming().failed().build(); + } else if (signal instanceof StreamingSubscriptionHasNext) { + topicPath = topicPathBuilder.streaming().hasNext().build(); + } else { + throw UnknownEventException.newBuilder(signal.getClass().getCanonicalName()).build(); + } + return topicPath; + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractPolicyMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractPolicyMappingStrategies.java index d0081a7f708..17ff8f195c3 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractPolicyMappingStrategies.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractPolicyMappingStrategies.java @@ -14,6 +14,7 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; +import java.util.Collection; import java.util.Map; import org.eclipse.ditto.base.model.json.Jsonifiable; @@ -36,6 +37,8 @@ import org.eclipse.ditto.policies.model.Subject; import org.eclipse.ditto.policies.model.SubjectId; import org.eclipse.ditto.policies.model.Subjects; +import org.eclipse.ditto.policies.model.signals.events.SubjectsDeletedPartially; +import org.eclipse.ditto.policies.model.signals.events.SubjectsModifiedPartially; import org.eclipse.ditto.protocol.Adaptable; import org.eclipse.ditto.protocol.JsonifiableMapper; import org.eclipse.ditto.protocol.MessagePath; @@ -68,6 +71,11 @@ protected static PolicyId policyIdFromTopicPath(final TopicPath topicPath) { return PolicyId.of(topicPath.getNamespace(), topicPath.getEntityName()); } + protected static PolicyId policyIdFrom(final Adaptable adaptable) { + final TopicPath topicPath = adaptable.getTopicPath(); + return PolicyId.of(topicPath.getNamespace(), topicPath.getEntityName()); + } + /** * Policy from policy. * @@ -248,6 +256,32 @@ protected static Subjects subjectsFrom(final Adaptable adaptable) { return PoliciesModelFactory.newSubjects(value); } + /** + * Subjects that are modified indexed by their policy entry labels. + * + * @param adaptable the adaptable + * @return the subjects + */ + protected static Map> activatedSubjectsFrom(final Adaptable adaptable) { + final JsonObject value = getValueFromPayload(adaptable); + return SubjectsModifiedPartially.modifiedSubjectsFromJson( + value.getValueOrThrow(SubjectsModifiedPartially.JSON_MODIFIED_SUBJECTS) + ); + } + + /** + * Subjects that are modified indexed by their policy entry labels. + * + * @param adaptable the adaptable + * @return the subjects + */ + protected static Map> deletedSubjectIdsFrom(final Adaptable adaptable) { + final JsonObject value = getValueFromPayload(adaptable); + return SubjectsDeletedPartially.deletedSubjectsFromJson( + value.getValueOrThrow(SubjectsDeletedPartially.JSON_DELETED_SUBJECT_IDS) + ); + } + /** * @throws NullPointerException if the value is null. */ diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractStreamingSubscriptionMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractStreamingSubscriptionMappingStrategies.java new file mode 100644 index 00000000000..554cbf946d2 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/AbstractStreamingSubscriptionMappingStrategies.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mappingstrategies; + +import java.util.Map; +import java.util.Optional; + +import org.eclipse.ditto.base.model.entity.id.NamespacedEntityId; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.json.Jsonifiable; +import org.eclipse.ditto.base.model.signals.WithStreamingSubscriptionId; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonMissingFieldException; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.JsonifiableMapper; +import org.eclipse.ditto.protocol.Payload; +import org.eclipse.ditto.protocol.TopicPath; + +/** + * Provides helper methods to map from {@link Adaptable}s to streaming subscription commands and events. + * + * @param the type of the mapped signals + */ +abstract class AbstractStreamingSubscriptionMappingStrategies> + extends AbstractMappingStrategies { + + protected AbstractStreamingSubscriptionMappingStrategies(final Map> mappingStrategies) { + super(mappingStrategies); + } + + protected static NamespacedEntityId entityIdFrom(final Adaptable adaptable) { + final TopicPath topicPath = adaptable.getTopicPath(); + return NamespacedEntityId.of(topicPath.getGroup().getEntityType(), + topicPath.getNamespace() + ":" + topicPath.getEntityName()); + } + + protected static EntityType entityTypeFrom(final Adaptable adaptable) { + final TopicPath topicPath = adaptable.getTopicPath(); + return topicPath.getGroup().getEntityType(); + } + + protected static JsonPointer resourcePathFrom(final Adaptable adaptable) { + return adaptable.getPayload().getPath(); + } + + protected static String subscriptionIdFrom(final Adaptable adaptable) { + return adaptable.getPayload().getValue() + .map(value -> value.asObject().getValueOrThrow(WithStreamingSubscriptionId.JsonFields.SUBSCRIPTION_ID)) + .orElseThrow(() -> JsonMissingFieldException.newBuilder() + .fieldName(Payload.JsonFields.VALUE.getPointer()) + .build() + ); + } + + static Optional getFromValue(final Adaptable adaptable, final JsonFieldDefinition jsonFieldDefinition) { + return adaptable.getPayload().getValue().flatMap(value -> value.asObject().getValue(jsonFieldDefinition)); + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/MappingStrategiesFactory.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/MappingStrategiesFactory.java index 90da18949cf..22b7abf4fda 100644 --- a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/MappingStrategiesFactory.java +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/MappingStrategiesFactory.java @@ -43,6 +43,10 @@ public static PolicyAnnouncementMappingStrategies getPolicyAnnouncementMappingSt return PolicyAnnouncementMappingStrategies.getInstance(); } + public static PolicyEventMappingStrategies getPolicyEventMappingStrategies() { + return PolicyEventMappingStrategies.getInstance(); + } + public static ThingMergeCommandMappingStrategies getThingMergeCommandMappingStrategies() { return ThingMergeCommandMappingStrategies.getInstance(); } @@ -110,4 +114,13 @@ public static ConnectivityAnnouncementMappingStrategies getConnectivityAnnouncem return ConnectivityAnnouncementMappingStrategies.getInstance(); } + public static StreamingSubscriptionCommandMappingStrategies getStreamingSubscriptionCommandMappingStrategies() { + return StreamingSubscriptionCommandMappingStrategies.getInstance(); + } + + public static StreamingSubscriptionEventMappingStrategies getStreamingSubscriptionEventMappingStrategies( + final ErrorRegistry errorRegistry) { + return StreamingSubscriptionEventMappingStrategies.getInstance(errorRegistry); + } + } diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/PolicyEventMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/PolicyEventMappingStrategies.java new file mode 100644 index 00000000000..0dee6ad0aab --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/PolicyEventMappingStrategies.java @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mappingstrategies; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.json.JsonMissingFieldException; +import org.eclipse.ditto.policies.model.signals.events.PolicyCreated; +import org.eclipse.ditto.policies.model.signals.events.PolicyDeleted; +import org.eclipse.ditto.policies.model.signals.events.PolicyEntriesModified; +import org.eclipse.ditto.policies.model.signals.events.PolicyEntryCreated; +import org.eclipse.ditto.policies.model.signals.events.PolicyEntryDeleted; +import org.eclipse.ditto.policies.model.signals.events.PolicyEntryModified; +import org.eclipse.ditto.policies.model.signals.events.PolicyEvent; +import org.eclipse.ditto.policies.model.signals.events.PolicyModified; +import org.eclipse.ditto.policies.model.signals.events.ResourceCreated; +import org.eclipse.ditto.policies.model.signals.events.ResourceDeleted; +import org.eclipse.ditto.policies.model.signals.events.ResourceModified; +import org.eclipse.ditto.policies.model.signals.events.ResourcesModified; +import org.eclipse.ditto.policies.model.signals.events.SubjectCreated; +import org.eclipse.ditto.policies.model.signals.events.SubjectDeleted; +import org.eclipse.ditto.policies.model.signals.events.SubjectModified; +import org.eclipse.ditto.policies.model.signals.events.SubjectsDeletedPartially; +import org.eclipse.ditto.policies.model.signals.events.SubjectsModified; +import org.eclipse.ditto.policies.model.signals.events.SubjectsModifiedPartially; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.JsonifiableMapper; +import org.eclipse.ditto.protocol.Payload; + +/** + * Defines mapping strategies (map from signal type to JsonifiableMapper) for policy events. + */ +final class PolicyEventMappingStrategies extends AbstractPolicyMappingStrategies> { + + private static final PolicyEventMappingStrategies INSTANCE = new PolicyEventMappingStrategies(); + + private PolicyEventMappingStrategies() { + super(initMappingStrategies()); + } + + static PolicyEventMappingStrategies getInstance() { + return INSTANCE; + } + + private static Map>> initMappingStrategies() { + final Map>> mappingStrategies = new HashMap<>(); + addTopLevelEvents(mappingStrategies); + addPolicyEntriesEvents(mappingStrategies); + addPolicyEntryEvents(mappingStrategies); + addResourcesEvents(mappingStrategies); + addResourceEvents(mappingStrategies); + addSubjectsEvents(mappingStrategies); + addSubjectEvents(mappingStrategies); + return mappingStrategies; + } + + private static void addTopLevelEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(PolicyCreated.TYPE, + adaptable -> PolicyCreated.of(policyFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(PolicyModified.TYPE, + adaptable -> PolicyModified.of(policyFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(PolicyDeleted.TYPE, + adaptable -> PolicyDeleted.of(policyIdFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addPolicyEntriesEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(PolicyEntriesModified.TYPE, + adaptable -> PolicyEntriesModified.of(policyIdFrom(adaptable), + policyEntriesFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addPolicyEntryEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(PolicyEntryCreated.TYPE, + adaptable -> PolicyEntryCreated.of(policyIdFrom(adaptable), + policyEntryFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(PolicyEntryModified.TYPE, + adaptable -> PolicyEntryModified.of(policyIdFrom(adaptable), + policyEntryFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(PolicyEntryDeleted.TYPE, + adaptable -> PolicyEntryDeleted.of(policyIdFrom(adaptable), + labelFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addResourcesEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(ResourcesModified.TYPE, + adaptable -> ResourcesModified.of(policyIdFrom(adaptable), + labelFrom(adaptable), + resourcesFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addResourceEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(ResourceCreated.TYPE, + adaptable -> ResourceCreated.of(policyIdFrom(adaptable), + labelFrom(adaptable), + resourceFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(ResourceModified.TYPE, + adaptable -> ResourceModified.of(policyIdFrom(adaptable), + labelFrom(adaptable), + resourceFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(ResourceDeleted.TYPE, + adaptable -> ResourceDeleted.of(policyIdFrom(adaptable), + labelFrom(adaptable), + entryResourceKeyFromPath(adaptable.getPayload().getPath()), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addSubjectsEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(SubjectsModified.TYPE, + adaptable -> SubjectsModified.of(policyIdFrom(adaptable), + labelFrom(adaptable), + subjectsFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(SubjectsModifiedPartially.TYPE, + adaptable -> SubjectsModifiedPartially.of(policyIdFrom(adaptable), + activatedSubjectsFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(SubjectsDeletedPartially.TYPE, + adaptable -> SubjectsDeletedPartially.of(policyIdFrom(adaptable), + deletedSubjectIdsFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static void addSubjectEvents( + final Map>> mappingStrategies) { + mappingStrategies.put(SubjectCreated.TYPE, + adaptable -> SubjectCreated.of(policyIdFrom(adaptable), + labelFrom(adaptable), + subjectFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(SubjectModified.TYPE, + adaptable -> SubjectModified.of(policyIdFrom(adaptable), + labelFrom(adaptable), + subjectFrom(adaptable), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + mappingStrategies.put(SubjectDeleted.TYPE, + adaptable -> SubjectDeleted.of(policyIdFrom(adaptable), + labelFrom(adaptable), + entrySubjectIdFromPath(adaptable.getPayload().getPath()), + revisionFrom(adaptable), + timestampFrom(adaptable), + dittoHeadersFrom(adaptable), + metadataFrom(adaptable))); + } + + private static long revisionFrom(final Adaptable adaptable) { + return adaptable.getPayload().getRevision().orElseThrow(() -> JsonMissingFieldException.newBuilder() + .fieldName(Payload.JsonFields.REVISION.getPointer().toString()).build()); + } + + @Nullable + private static Instant timestampFrom(final Adaptable adaptable) { + return adaptable.getPayload().getTimestamp().orElse(null); + } + + @Nullable + private static Metadata metadataFrom(final Adaptable adaptable) { + return adaptable.getPayload().getMetadata().orElse(null); + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionCommandMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionCommandMappingStrategies.java new file mode 100644 index 00000000000..fb8881e7eb3 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionCommandMappingStrategies.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mappingstrategies; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.signals.commands.streaming.CancelStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.RequestFromStreamingSubscription; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.JsonifiableMapper; + +/** + * Defines mapping strategies (map from signal type to JsonifiableMapper) for streaming subscription commands. + */ +final class StreamingSubscriptionCommandMappingStrategies + extends AbstractStreamingSubscriptionMappingStrategies> { + + private static final StreamingSubscriptionCommandMappingStrategies INSTANCE = + new StreamingSubscriptionCommandMappingStrategies(); + + private StreamingSubscriptionCommandMappingStrategies() { + super(initMappingStrategies()); + } + + static StreamingSubscriptionCommandMappingStrategies getInstance() { + return INSTANCE; + } + + private static Map>> initMappingStrategies() { + + final Map>> mappingStrategies = new HashMap<>(); + + mappingStrategies.put(SubscribeForPersistedEvents.TYPE, + adaptable -> SubscribeForPersistedEvents.of(entityIdFrom(adaptable), + resourcePathFrom(adaptable), + fromHistoricalRevision(adaptable), + toHistoricalRevision(adaptable), + fromHistoricalTimestamp(adaptable), + toHistoricalTimestamp(adaptable), + dittoHeadersFrom(adaptable))); + mappingStrategies.put(CancelStreamingSubscription.TYPE, + adaptable -> CancelStreamingSubscription.of(entityIdFrom(adaptable), + resourcePathFrom(adaptable), + Objects.requireNonNull(subscriptionIdFrom(adaptable)), + dittoHeadersFrom(adaptable))); + mappingStrategies.put(RequestFromStreamingSubscription.TYPE, + adaptable -> RequestFromStreamingSubscription.of(entityIdFrom(adaptable), + resourcePathFrom(adaptable), + Objects.requireNonNull(subscriptionIdFrom(adaptable)), + demandFrom(adaptable), + dittoHeadersFrom(adaptable))); + + return mappingStrategies; + } + + @Nullable + private static Long fromHistoricalRevision(final Adaptable adaptable) { + return getFromValue(adaptable, SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_REVISION) + .orElse(null); + } + + @Nullable + private static Long toHistoricalRevision(final Adaptable adaptable) { + return getFromValue(adaptable, SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_REVISION) + .orElse(null); + } + + @Nullable + private static Instant fromHistoricalTimestamp(final Adaptable adaptable) { + return getFromValue(adaptable, SubscribeForPersistedEvents.JsonFields.JSON_FROM_HISTORICAL_TIMESTAMP) + .map(Instant::parse) + .orElse(null); + } + + @Nullable + private static Instant toHistoricalTimestamp(final Adaptable adaptable) { + return getFromValue(adaptable, SubscribeForPersistedEvents.JsonFields.JSON_TO_HISTORICAL_TIMESTAMP) + .map(Instant::parse) + .orElse(null); + } + + private static long demandFrom(final Adaptable adaptable) { + return getFromValue(adaptable, RequestFromStreamingSubscription.JsonFields.DEMAND).orElse(0L); + } + +} diff --git a/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionEventMappingStrategies.java b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionEventMappingStrategies.java new file mode 100644 index 00000000000..07bfa9abe56 --- /dev/null +++ b/protocol/src/main/java/org/eclipse/ditto/protocol/mappingstrategies/StreamingSubscriptionEventMappingStrategies.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.protocol.mappingstrategies; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.ErrorRegistry; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionCreated; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionEvent; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionFailed; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionHasNext; +import org.eclipse.ditto.json.JsonParseException; +import org.eclipse.ditto.json.JsonValue; +import org.eclipse.ditto.protocol.Adaptable; +import org.eclipse.ditto.protocol.JsonifiableMapper; +import org.eclipse.ditto.protocol.adapter.AbstractErrorResponseAdapter; + +/** + * Defines mapping strategies (map from signal type to JsonifiableMapper) for streaming subscription events. + */ +final class StreamingSubscriptionEventMappingStrategies + extends AbstractStreamingSubscriptionMappingStrategies> { + + private StreamingSubscriptionEventMappingStrategies(final ErrorRegistry errorRegistry) { + super(initMappingStrategies(errorRegistry)); + } + + static StreamingSubscriptionEventMappingStrategies getInstance(final ErrorRegistry errorRegistry) { + return new StreamingSubscriptionEventMappingStrategies(errorRegistry); + } + + private static Map>> initMappingStrategies( + final ErrorRegistry errorRegistry) { + + final Map>> mappingStrategies = new HashMap<>(); + + mappingStrategies.put(StreamingSubscriptionCreated.TYPE, + adaptable -> StreamingSubscriptionCreated.of(Objects.requireNonNull(subscriptionIdFrom(adaptable)), + entityIdFrom(adaptable), dittoHeadersFrom(adaptable))); + mappingStrategies.put(StreamingSubscriptionComplete.TYPE, + adaptable -> StreamingSubscriptionComplete.of(Objects.requireNonNull(subscriptionIdFrom(adaptable)), + entityIdFrom(adaptable), dittoHeadersFrom(adaptable))); + mappingStrategies.put(StreamingSubscriptionFailed.TYPE, + adaptable -> StreamingSubscriptionFailed.of(Objects.requireNonNull(subscriptionIdFrom(adaptable)), + entityIdFrom(adaptable), errorFrom(adaptable, errorRegistry), dittoHeadersFrom(adaptable))); + mappingStrategies.put(StreamingSubscriptionHasNext.TYPE, + adaptable -> StreamingSubscriptionHasNext.of(Objects.requireNonNull(subscriptionIdFrom(adaptable)), + entityIdFrom(adaptable),itemFrom(adaptable), dittoHeadersFrom(adaptable))); + + return mappingStrategies; + } + + private static JsonValue itemFrom(final Adaptable adaptable) { + return getFromValue(adaptable, StreamingSubscriptionHasNext.JsonFields.ITEM).orElseGet(JsonValue::nullLiteral); + } + + private static DittoRuntimeException errorFrom(final Adaptable adaptable, final ErrorRegistry errorRegistry) { + return getFromValue(adaptable, StreamingSubscriptionFailed.JsonFields.ERROR) + .map(error -> + AbstractErrorResponseAdapter.parseWithErrorRegistry(error, DittoHeaders.empty(), errorRegistry)) + .orElseThrow(() -> JsonParseException.newBuilder().build()); + } + +} diff --git a/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapterParameterizedTest.java b/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapterParameterizedTest.java index cae6597b5fa..c99d8507e33 100644 --- a/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapterParameterizedTest.java +++ b/protocol/src/test/java/org/eclipse/ditto/protocol/adapter/DittoProtocolAdapterParameterizedTest.java @@ -212,6 +212,11 @@ public void setUp() { final ConnectivityCommandAdapterProvider connectivityCommandAdapterProvider = mock(ConnectivityCommandAdapterProvider.class); + final StreamingSubscriptionCommandAdapter streamingSubscriptionCommandAdapter = + mock(StreamingSubscriptionCommandAdapter.class); + final StreamingSubscriptionEventAdapter streamingSubscriptionEventAdapter = + mock(StreamingSubscriptionEventAdapter.class); + when(thingCommandAdapterProvider.getQueryCommandAdapter()) .thenReturn(thingQueryCommandAdapter); when(thingCommandAdapterProvider.getQueryCommandResponseAdapter()) @@ -247,9 +252,11 @@ public void setUp() { .thenReturn(policyErrorResponseAdapter); final AdapterResolver adapterResolver = new DefaultAdapterResolver(thingCommandAdapterProvider, - policyCommandAdapterProvider, connectivityCommandAdapterProvider, acknowledgementAdapterProvider); + policyCommandAdapterProvider, connectivityCommandAdapterProvider, acknowledgementAdapterProvider, + streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter); underTest = DittoProtocolAdapter.newInstance(HeaderTranslator.empty(), thingCommandAdapterProvider, policyCommandAdapterProvider, connectivityCommandAdapterProvider, acknowledgementAdapterProvider, + streamingSubscriptionCommandAdapter, streamingSubscriptionEventAdapter, adapterResolver); reset(thingQueryCommandAdapter); diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/InvalidThingFieldSelectionException.java b/things/model/src/main/java/org/eclipse/ditto/things/model/InvalidThingFieldSelectionException.java index 6b32b00180d..efcbd64f6c4 100644 --- a/things/model/src/main/java/org/eclipse/ditto/things/model/InvalidThingFieldSelectionException.java +++ b/things/model/src/main/java/org/eclipse/ditto/things/model/InvalidThingFieldSelectionException.java @@ -18,12 +18,12 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; -import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.base.model.common.HttpStatus; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; /** * Thrown when a {@link ThingFieldSelector} was not valid. @@ -34,9 +34,9 @@ public final class InvalidThingFieldSelectionException extends DittoRuntimeException implements ThingException { private static final String DEFAULT_MESSAGE_TEMPLATE = "Thing field selection <{0}> was not valid."; - private static final String DEFAULT_DESCRIPTION = "Please provide a comma separated List of valid Thing fields." + + private static final String DEFAULT_DESCRIPTION = "Please provide a comma separated List of valid Thing fields. " + "Make sure that you did not use a space after a comma. Valid Fields are: " + - ThingFieldSelector.SELECTABLE_FIELDS.toString(); + ThingFieldSelector.SELECTABLE_FIELDS; static final String ERROR_CODE = ERROR_CODE_PREFIX + "field.selection.invalid"; diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/ThingFieldSelector.java b/things/model/src/main/java/org/eclipse/ditto/things/model/ThingFieldSelector.java index 835d8efd904..81c3e55e8a5 100644 --- a/things/model/src/main/java/org/eclipse/ditto/things/model/ThingFieldSelector.java +++ b/things/model/src/main/java/org/eclipse/ditto/things/model/ThingFieldSelector.java @@ -38,7 +38,7 @@ public final class ThingFieldSelector implements JsonFieldSelector { .withoutUrlDecoding() .build(); static final List SELECTABLE_FIELDS = Arrays.asList("thingId", "policyId", "definition", - "_namespace", "_revision", "_created", "_modified", "_metadata", "_policy", + "_namespace", "_revision", "_created", "_modified", "_metadata", "_policy", "_context", "_context(/[^,]+)?", "features(/[^,]+)?", "attributes(/[^,]+)?"); private static final String KNOWN_FIELDS_REGEX = "/?(" + String.join("|", SELECTABLE_FIELDS) + ")"; private static final String FIELD_SELECTION_REGEX = "^" + KNOWN_FIELDS_REGEX + "(," + KNOWN_FIELDS_REGEX + ")*$"; diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/WithThingId.java b/things/model/src/main/java/org/eclipse/ditto/things/model/WithThingId.java index 67c1ce14736..3b156c9d299 100644 --- a/things/model/src/main/java/org/eclipse/ditto/things/model/WithThingId.java +++ b/things/model/src/main/java/org/eclipse/ditto/things/model/WithThingId.java @@ -16,7 +16,7 @@ /** * Implementations of this interface are associated to a {@code Thing} identified by the value - * returned from {@link #getEntityId()} ()}. + * returned from {@link #getEntityId()}. */ public interface WithThingId extends WithEntityId { diff --git a/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingHistoryNotAccessibleException.java b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingHistoryNotAccessibleException.java new file mode 100755 index 00000000000..e06eb9e1a1e --- /dev/null +++ b/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/exceptions/ThingHistoryNotAccessibleException.java @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.things.model.signals.commands.exceptions; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.net.URI; +import java.text.MessageFormat; +import java.time.Instant; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.things.model.ThingException; +import org.eclipse.ditto.things.model.ThingId; + +/** + * Thrown if historical data of the Thing was either not present in Ditto at all or if the requester had insufficient + * permissions to access it. + * + * @since 3.2.0 + */ +@Immutable +@JsonParsableException(errorCode = ThingHistoryNotAccessibleException.ERROR_CODE) +public final class ThingHistoryNotAccessibleException extends DittoRuntimeException implements ThingException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "thing.history.notfound"; + + private static final String MESSAGE_TEMPLATE = + "The Thing with ID ''{0}'' at revision ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String MESSAGE_TEMPLATE_TS = + "The Thing with ID ''{0}'' at timestamp ''{1}'' could not be found or requester had insufficient " + + "permissions to access it."; + + private static final String DEFAULT_DESCRIPTION = + "Check if the ID of your requested Thing was correct, you have sufficient permissions and ensure that the " + + "asked for revision/timestamp does not exceed the history-retention-duration."; + + private static final long serialVersionUID = 8883736111094383234L; + + private ThingHistoryNotAccessibleException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + super(ERROR_CODE, HttpStatus.NOT_FOUND, dittoHeaders, message, description, cause, href); + } + + private static String getMessage(final ThingId thingId, final long revision) { + checkNotNull(thingId, "thingId"); + return MessageFormat.format(MESSAGE_TEMPLATE, String.valueOf(thingId), String.valueOf(revision)); + } + + private static String getMessage(final ThingId thingId, final Instant timestamp) { + checkNotNull(thingId, "thingId"); + checkNotNull(timestamp, "timestamp"); + return MessageFormat.format(MESSAGE_TEMPLATE_TS, String.valueOf(thingId), timestamp.toString()); + } + + /** + * A mutable builder for a {@code ThingHistoryNotAccessibleException}. + * + * @param thingId the ID of the thing. + * @param revision the asked for revision of the thing. + * @return the builder. + * @throws NullPointerException if {@code thingId} is {@code null}. + */ + public static Builder newBuilder(final ThingId thingId, final long revision) { + return new Builder(thingId, revision); + } + + /** + * A mutable builder for a {@code ThingHistoryNotAccessibleException}. + * + * @param thingId the ID of the thing. + * @param timestamp the asked for timestamp of the thing. + * @return the builder. + * @throws NullPointerException if {@code thingId} is {@code null}. + */ + public static Builder newBuilder(final ThingId thingId, final Instant timestamp) { + return new Builder(thingId, timestamp); + } + + /** + * Constructs a new {@code ThingHistoryNotAccessibleException} object with given message. + * + * @param message detail message. This message can be later retrieved by the {@link #getMessage()} method. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new ThingHistoryNotAccessibleException. + * @throws NullPointerException if {@code dittoHeaders} is {@code null}. + */ + public static ThingHistoryNotAccessibleException fromMessage(@Nullable final String message, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromMessage(message, dittoHeaders, new Builder()); + } + + /** + * Constructs a new {@code ThingHistoryNotAccessibleException} object with the exception message extracted from the given + * JSON object. + * + * @param jsonObject the JSON to read the {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new ThingHistoryNotAccessibleException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static ThingHistoryNotAccessibleException fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link org.eclipse.ditto.things.model.signals.commands.exceptions.ThingHistoryNotAccessibleException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() { + description(DEFAULT_DESCRIPTION); + } + + private Builder(final ThingId thingId, final long revision) { + this(); + message(ThingHistoryNotAccessibleException.getMessage(thingId, revision)); + } + + private Builder(final ThingId thingId, final Instant timestamp) { + this(); + message(ThingHistoryNotAccessibleException.getMessage(thingId, timestamp)); + } + + @Override + protected ThingHistoryNotAccessibleException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new ThingHistoryNotAccessibleException(dittoHeaders, message, description, cause, href); + } + + } + +} diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/DefaultThingConfig.java b/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/DefaultThingConfig.java index 00c0774e8f0..649e08d1ca1 100644 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/DefaultThingConfig.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/DefaultThingConfig.java @@ -23,7 +23,9 @@ import org.eclipse.ditto.internal.utils.config.ScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultActivityCheckConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultEventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.DefaultSnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.CleanupConfig; @@ -41,6 +43,7 @@ public final class DefaultThingConfig implements ThingConfig { private final SupervisorConfig supervisorConfig; private final ActivityCheckConfig activityCheckConfig; private final SnapshotConfig snapshotConfig; + private final EventConfig eventConfig; private final CleanupConfig cleanupConfig; private DefaultThingConfig(final ScopedConfig scopedConfig) { @@ -48,6 +51,7 @@ private DefaultThingConfig(final ScopedConfig scopedConfig) { supervisorConfig = DefaultSupervisorConfig.of(scopedConfig); activityCheckConfig = DefaultActivityCheckConfig.of(scopedConfig); snapshotConfig = DefaultSnapshotConfig.of(scopedConfig); + eventConfig = DefaultEventConfig.of(scopedConfig); cleanupConfig = CleanupConfig.of(scopedConfig); } @@ -82,6 +86,11 @@ public CleanupConfig getCleanupConfig() { return cleanupConfig; } + @Override + public EventConfig getEventConfig() { + return eventConfig; + } + @Override public Duration getShutdownTimeout() { return shutdownTimeout; @@ -99,13 +108,15 @@ public boolean equals(final Object o) { return Objects.equals(supervisorConfig, that.supervisorConfig) && Objects.equals(activityCheckConfig, that.activityCheckConfig) && Objects.equals(snapshotConfig, that.snapshotConfig) && + Objects.equals(eventConfig, that.eventConfig) && Objects.equals(cleanupConfig, that.cleanupConfig) && Objects.equals(shutdownTimeout, that.shutdownTimeout); } @Override public int hashCode() { - return Objects.hash(supervisorConfig, activityCheckConfig, snapshotConfig, cleanupConfig, shutdownTimeout); + return Objects.hash(supervisorConfig, activityCheckConfig, snapshotConfig, eventConfig, cleanupConfig, + shutdownTimeout); } @Override @@ -114,6 +125,7 @@ public String toString() { "supervisorConfig=" + supervisorConfig + ", activityCheckConfig=" + activityCheckConfig + ", snapshotConfig=" + snapshotConfig + + ", eventConfig=" + eventConfig + ", cleanupConfig=" + cleanupConfig + ", shutdownTimeout=" + shutdownTimeout + "]"; diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/ThingConfig.java b/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/ThingConfig.java index 544dd82a3cd..d9dc9936518 100644 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/ThingConfig.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/common/config/ThingConfig.java @@ -18,6 +18,7 @@ import org.eclipse.ditto.base.service.config.supervision.WithSupervisorConfig; import org.eclipse.ditto.internal.utils.config.KnownConfigValue; +import org.eclipse.ditto.internal.utils.persistence.mongo.config.EventConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.WithSnapshotConfig; import org.eclipse.ditto.internal.utils.persistentactors.cleanup.WithCleanupConfig; @@ -29,6 +30,13 @@ public interface ThingConfig extends WithSupervisorConfig, WithActivityCheckConfig, WithSnapshotConfig, WithCleanupConfig { + /** + * Returns the config of the thing event journal behaviour. + * + * @return the config. + */ + EventConfig getEventConfig(); + /** * Get the timeout waiting for responses and acknowledgements during coordinated shutdown. * diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/StreamRequestingCommandEnforcement.java b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/StreamRequestingCommandEnforcement.java new file mode 100644 index 00000000000..d837476ffee --- /dev/null +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/StreamRequestingCommandEnforcement.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.things.service.enforcement; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.base.model.signals.commands.CommandResponse; +import org.eclipse.ditto.base.model.signals.commands.streaming.StreamingSubscriptionCommand; +import org.eclipse.ditto.policies.api.Permission; +import org.eclipse.ditto.policies.enforcement.AbstractEnforcementReloaded; +import org.eclipse.ditto.policies.enforcement.EnforcementReloaded; +import org.eclipse.ditto.policies.enforcement.PolicyEnforcer; +import org.eclipse.ditto.policies.model.Permissions; +import org.eclipse.ditto.policies.model.ResourceKey; +import org.eclipse.ditto.things.model.ThingId; +import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingNotAccessibleException; + +/** + * Authorizes {@link StreamingSubscriptionCommand}s and filters {@link CommandResponse}s. + */ +final class StreamRequestingCommandEnforcement + extends AbstractEnforcementReloaded, CommandResponse> + implements ThingEnforcementStrategy { + + @Override + public boolean isApplicable(final Signal signal) { + return signal instanceof StreamingSubscriptionCommand; + } + + @Override + public boolean responseIsApplicable(final CommandResponse commandResponse) { + return false; + } + + @Override + public , R extends CommandResponse> EnforcementReloaded getEnforcement() { + return (EnforcementReloaded) this; + } + + @Override + public CompletionStage> authorizeSignal(final StreamingSubscriptionCommand signal, + final PolicyEnforcer policyEnforcer) { + + final ResourceKey resourceKey = ResourceKey.newInstance(signal.getResourceType(), signal.getResourcePath()); + if (policyEnforcer.getEnforcer().hasUnrestrictedPermissions(resourceKey, + signal.getDittoHeaders().getAuthorizationContext(), Permissions.newInstance(Permission.READ))) { + return CompletableFuture.completedStage( + ThingCommandEnforcement.addEffectedReadSubjectsToThingSignal(signal, policyEnforcer.getEnforcer()) + ); + } else { + return CompletableFuture.failedStage( + ThingNotAccessibleException.newBuilder(ThingId.of(signal.getEntityId())) + .dittoHeaders(signal.getDittoHeaders()) + .build()); + } + } + + @Override + public CompletionStage> authorizeSignalWithMissingEnforcer( + final StreamingSubscriptionCommand signal) { + + return CompletableFuture.failedStage(ThingNotAccessibleException.newBuilder(ThingId.of(signal.getEntityId())) + .dittoHeaders(signal.getDittoHeaders()) + .build()); + } + + @Override + public boolean shouldFilterCommandResponse(final CommandResponse commandResponse) { + return false; + } + + @Override + public CompletionStage> filterResponse(final CommandResponse commandResponse, + final PolicyEnforcer policyEnforcer) { + return CompletableFuture.completedStage(commandResponse); + } +} diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcement.java b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcement.java index a2462c528fe..ff969d271e7 100644 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcement.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingEnforcement.java @@ -40,7 +40,8 @@ public ThingEnforcement(final ActorRef policiesShardRegion, final ActorSystem ac enforcementStrategies = List.of( new LiveSignalEnforcement(), - new ThingCommandEnforcement(actorSystem, policiesShardRegion, enforcementConfig) + new ThingCommandEnforcement(actorSystem, policiesShardRegion, enforcementConfig), + new StreamRequestingCommandEnforcement() ); } diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActor.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActor.java index 9e682bc0b88..5245df33416 100755 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActor.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActor.java @@ -12,6 +12,8 @@ */ package org.eclipse.ditto.things.service.persistence.actors; +import java.time.Instant; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.acks.DittoAcknowledgementLabel; @@ -24,6 +26,7 @@ import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.ActivityCheckConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.config.SnapshotConfig; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceActor; import org.eclipse.ditto.internal.utils.persistentactors.commands.CommandStrategy; import org.eclipse.ditto.internal.utils.persistentactors.commands.DefaultContext; @@ -38,6 +41,7 @@ import org.eclipse.ditto.things.model.ThingId; import org.eclipse.ditto.things.model.ThingLifecycle; import org.eclipse.ditto.things.model.ThingsModelFactory; +import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingHistoryNotAccessibleException; import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingNotAccessibleException; import org.eclipse.ditto.things.model.signals.commands.modify.CreateThing; import org.eclipse.ditto.things.model.signals.commands.query.RetrieveThing; @@ -83,10 +87,11 @@ public final class ThingPersistenceActor @SuppressWarnings("unused") private ThingPersistenceActor(final ThingId thingId, + final MongoReadJournal mongoReadJournal, final DistributedPub> distributedPub, @Nullable final ActorRef searchShardRegionProxy) { - super(thingId); + super(thingId, mongoReadJournal); final DittoThingsConfig thingsConfig = DittoThingsConfig.of( DefaultScopedConfig.dittoScoped(getContext().getSystem().settings().config()) ); @@ -99,14 +104,17 @@ private ThingPersistenceActor(final ThingId thingId, * Creates Akka configuration object {@link Props} for this ThingPersistenceActor. * * @param thingId the Thing ID this Actor manages. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the thing. * @param distributedPub the distributed-pub access to publish thing events. * @return the Akka configuration Props object */ public static Props props(final ThingId thingId, + final MongoReadJournal mongoReadJournal, final DistributedPub> distributedPub, @Nullable final ActorRef searchShardRegionProxy) { - return Props.create(ThingPersistenceActor.class, thingId, distributedPub, searchShardRegionProxy); + return Props.create(ThingPersistenceActor.class, thingId, mongoReadJournal, distributedPub, + searchShardRegionProxy); } @Override @@ -193,6 +201,16 @@ protected DittoRuntimeExceptionBuilder newNotAccessibleExceptionBuilder() { return ThingNotAccessibleException.newBuilder(entityId); } + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final long revision) { + return ThingHistoryNotAccessibleException.newBuilder(entityId, revision); + } + + @Override + protected DittoRuntimeExceptionBuilder newHistoryNotAccessibleExceptionBuilder(final Instant timestamp) { + return ThingHistoryNotAccessibleException.newBuilder(entityId, timestamp); + } + @Override protected void recoveryCompleted(final RecoveryCompleted event) { if (entity != null) { diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorPropsFactory.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorPropsFactory.java index 7845e202f82..a5d5777896d 100644 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorPropsFactory.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorPropsFactory.java @@ -14,8 +14,9 @@ import javax.annotation.Nullable; -import org.eclipse.ditto.things.model.ThingId; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; +import org.eclipse.ditto.things.model.ThingId; import org.eclipse.ditto.things.model.signals.events.ThingEvent; import akka.actor.ActorRef; @@ -31,10 +32,11 @@ public interface ThingPersistenceActorPropsFactory { * Create Props of thing-persistence-actor from thing ID and distributed-pub access for event publishing. * * @param thingId the thing ID. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the thing. * @param distributedPub the distributed-pub access. * @param searchShardRegionProxy the proxy of the shard region of search updaters. * @return Props of the thing-persistence-actor. */ - Props props(ThingId thingId, DistributedPub> distributedPub, - @Nullable final ActorRef searchShardRegionProxy); + Props props(ThingId thingId, MongoReadJournal mongoReadJournal, DistributedPub> distributedPub, + @Nullable ActorRef searchShardRegionProxy); } diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingSupervisorActor.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingSupervisorActor.java index d7c732b1c71..6e93323c2ea 100755 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingSupervisorActor.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingSupervisorActor.java @@ -34,6 +34,7 @@ import org.eclipse.ditto.internal.utils.cluster.StopShardedActor; import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.namespaces.BlockedNamespaces; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.persistentactors.AbstractPersistenceSupervisor; import org.eclipse.ditto.internal.utils.persistentactors.TargetActorWithMessage; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; @@ -102,9 +103,10 @@ private ThingSupervisorActor(final ActorRef pubSubMediator, @Nullable final ThingPersistenceActorPropsFactory thingPersistenceActorPropsFactory, @Nullable final ActorRef thingPersistenceActorRef, @Nullable final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { - super(blockedNamespaces, DEFAULT_LOCAL_ASK_TIMEOUT); + super(blockedNamespaces, mongoReadJournal, DEFAULT_LOCAL_ASK_TIMEOUT); this.policyEnforcerProvider = policyEnforcerProvider; this.pubSubMediator = pubSubMediator; @@ -174,6 +176,7 @@ private ThingSupervisorActor(final ActorRef pubSubMediator, * @param propsFactory factory for creating Props to be used for creating * @param blockedNamespaces the blocked namespaces functionality to retrieve/subscribe for blocked namespaces. * @param policyEnforcerProvider used to load the policy enforcer. + * @param mongoReadJournal the ReadJournal used for gaining access to historical values of the thing. * @return the {@link Props} to create this actor. */ public static Props props(final ActorRef pubSubMediator, @@ -181,11 +184,12 @@ public static Props props(final ActorRef pubSubMediator, final LiveSignalPub liveSignalPub, final ThingPersistenceActorPropsFactory propsFactory, @Nullable final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { return Props.create(ThingSupervisorActor.class, pubSubMediator, null, distributedPubThingEventsForTwin, liveSignalPub, propsFactory, null, blockedNamespaces, - policyEnforcerProvider); + policyEnforcerProvider, mongoReadJournal); } /** @@ -197,11 +201,12 @@ public static Props props(final ActorRef pubSubMediator, final LiveSignalPub liveSignalPub, final ThingPersistenceActorPropsFactory propsFactory, @Nullable final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { return Props.create(ThingSupervisorActor.class, pubSubMediator, policiesShardRegion, distributedPubThingEventsForTwin, liveSignalPub, propsFactory, null, blockedNamespaces, - policyEnforcerProvider); + policyEnforcerProvider, mongoReadJournal); } /** @@ -213,11 +218,12 @@ public static Props props(final ActorRef pubSubMediator, final LiveSignalPub liveSignalPub, final ActorRef thingsPersistenceActor, @Nullable final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { return Props.create(ThingSupervisorActor.class, pubSubMediator, policiesShardRegion, distributedPubThingEventsForTwin, liveSignalPub, null, thingsPersistenceActor, blockedNamespaces, - policyEnforcerProvider); + policyEnforcerProvider, mongoReadJournal); } @Override @@ -317,7 +323,7 @@ protected ThingId getEntityId() throws Exception { @Override protected Props getPersistenceActorProps(final ThingId entityId) { assert thingPersistenceActorPropsFactory != null; - return thingPersistenceActorPropsFactory.props(entityId, distributedPubThingEventsForTwin, + return thingPersistenceActorPropsFactory.props(entityId, mongoReadJournal, distributedPubThingEventsForTwin, searchShardRegionProxy); } diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingsPersistenceStreamingActorCreator.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingsPersistenceStreamingActorCreator.java index bef308fc3e1..bae2ae9d751 100644 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingsPersistenceStreamingActorCreator.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/ThingsPersistenceStreamingActorCreator.java @@ -31,9 +31,9 @@ public final class ThingsPersistenceStreamingActorCreator { /** - * The name of the snapshot streaming actor. + * The name of the streaming actor. */ - public static final String SNAPSHOT_STREAMING_ACTOR_NAME = THINGS_PERSISTENCE_STREAMING_ACTOR_NAME; + public static final String STREAMING_ACTOR_NAME = THINGS_PERSISTENCE_STREAMING_ACTOR_NAME; private static final Pattern PERSISTENCE_ID_PATTERN = Pattern.compile(ThingPersistenceActor.PERSISTENCE_ID_PREFIX); @@ -42,16 +42,16 @@ private ThingsPersistenceStreamingActorCreator() { } /** - * Create an actor that streams from the snapshot store. + * Create an actor that streams from the snapshot store and the event journal. * * @param actorCreator function to create a named actor with. * @return a reference of the created actor. */ - public static ActorRef startSnapshotStreamingActor(final BiFunction actorCreator) { + public static ActorRef startPersistenceStreamingActor(final BiFunction actorCreator) { final var props = SnapshotStreamingActor.props(ThingsPersistenceStreamingActorCreator::pid2EntityId, ThingsPersistenceStreamingActorCreator::entityId2Pid); - return actorCreator.apply(SNAPSHOT_STREAMING_ACTOR_NAME, props); + return actorCreator.apply(STREAMING_ACTOR_NAME, props); } private static ThingId pid2EntityId(final String pid) { diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/events/ThingEventStrategies.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/events/ThingEventStrategies.java index 3cf0d4e8e41..42e6ca22799 100644 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/events/ThingEventStrategies.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/actors/strategies/events/ThingEventStrategies.java @@ -14,8 +14,8 @@ import javax.annotation.concurrent.Immutable; -import org.eclipse.ditto.things.model.Thing; import org.eclipse.ditto.internal.utils.persistentactors.events.AbstractEventStrategies; +import org.eclipse.ditto.things.model.Thing; import org.eclipse.ditto.things.model.signals.events.AttributeCreated; import org.eclipse.ditto.things.model.signals.events.AttributeDeleted; import org.eclipse.ditto.things.model.signals.events.AttributeModified; @@ -54,7 +54,7 @@ import org.eclipse.ditto.things.model.signals.events.ThingModified; /** - * This Singleton strategy handles all {@link org.eclipse.ditto.things.model.signals.events.ThingEvent}s. + * This Singleton strategy handles all {@link ThingEvent}s. */ @Immutable public final class ThingEventStrategies extends AbstractEventStrategies, Thing> { diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapter.java b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapter.java index 0f3e367ddfd..ba7d45a8ece 100644 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapter.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapter.java @@ -12,18 +12,17 @@ */ package org.eclipse.ditto.things.service.persistence.serializer; -import javax.annotation.Nullable; - -import org.eclipse.ditto.base.model.json.FieldType; -import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.events.Event; import org.eclipse.ditto.base.model.signals.events.GlobalEventRegistry; +import org.eclipse.ditto.base.service.config.DittoServiceConfig; +import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.internal.utils.persistence.mongo.AbstractMongoEventAdapter; -import org.eclipse.ditto.internal.utils.persistence.mongo.DittoBsonJson; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.policies.model.Policy; import org.eclipse.ditto.things.model.signals.events.ThingEvent; +import org.eclipse.ditto.things.service.common.config.DefaultThingConfig; import akka.actor.ExtendedActorSystem; @@ -36,29 +35,16 @@ public final class ThingMongoEventAdapter extends AbstractMongoEventAdapter event, final JsonObject jsonObject) { + return super.performToJournalMigration(event, jsonObject) .remove(POLICY_IN_THING_EVENT_PAYLOAD); // remove the policy entries from thing event payload } - @Override - public Object toJournal(final Object event) { - if (event instanceof Event theEvent) { - final JsonSchemaVersion schemaVersion = theEvent.getImplementedSchemaVersion(); - final JsonObject jsonObject = - theEvent.toJson(schemaVersion, FieldType.regularOrSpecial()) - // remove the policy entries from thing event payload - .remove(POLICY_IN_THING_EVENT_PAYLOAD); - final DittoBsonJson dittoBsonJson = DittoBsonJson.getInstance(); - return dittoBsonJson.parse(jsonObject); - } else { - throw new IllegalArgumentException("Unable to toJournal a non-'Event' object! Was: " + event.getClass()); - } - } - } diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/starter/DefaultThingPersistenceActorPropsFactory.java b/things/service/src/main/java/org/eclipse/ditto/things/service/starter/DefaultThingPersistenceActorPropsFactory.java index 91570851eda..abca24f39fc 100644 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/starter/DefaultThingPersistenceActorPropsFactory.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/starter/DefaultThingPersistenceActorPropsFactory.java @@ -17,6 +17,7 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.things.model.ThingId; import org.eclipse.ditto.things.model.signals.events.ThingEvent; @@ -51,9 +52,10 @@ static DefaultThingPersistenceActorPropsFactory of(final ActorSystem actorSystem } @Override - public Props props(final ThingId thingId, final DistributedPub> distributedPub, + public Props props(final ThingId thingId, final MongoReadJournal mongoReadJournal, + final DistributedPub> distributedPub, @Nullable final ActorRef searchShardRegionProxy) { argumentNotEmpty(thingId); - return ThingPersistenceActor.props(thingId, distributedPub, searchShardRegionProxy); + return ThingPersistenceActor.props(thingId, mongoReadJournal, distributedPub, searchShardRegionProxy); } } diff --git a/things/service/src/main/java/org/eclipse/ditto/things/service/starter/ThingsRootActor.java b/things/service/src/main/java/org/eclipse/ditto/things/service/starter/ThingsRootActor.java index 3c6fe00e7c4..83dbd50e29a 100755 --- a/things/service/src/main/java/org/eclipse/ditto/things/service/starter/ThingsRootActor.java +++ b/things/service/src/main/java/org/eclipse/ditto/things/service/starter/ThingsRootActor.java @@ -88,12 +88,14 @@ private ThingsRootActor(final ThingsConfig thingsConfig, final BlockedNamespaces blockedNamespaces = BlockedNamespaces.of(actorSystem); final PolicyEnforcerProvider policyEnforcerProvider = PolicyEnforcerProviderExtension.get(actorSystem).getPolicyEnforcerProvider(); + final var mongoReadJournal = newMongoReadJournal(thingsConfig.getMongoDbConfig(), actorSystem); final Props thingSupervisorActorProps = getThingSupervisorActorProps(pubSubMediator, distributedPubThingEventsForTwin, liveSignalPub, propsFactory, blockedNamespaces, - policyEnforcerProvider + policyEnforcerProvider, + mongoReadJournal ); final ActorRef thingsShardRegion = @@ -128,10 +130,9 @@ private ThingsRootActor(final ThingsConfig thingsConfig, DefaultHealthCheckingActorFactory.props(healthCheckingActorOptions, MongoHealthChecker.props())); final ActorRef snapshotStreamingActor = - ThingsPersistenceStreamingActorCreator.startSnapshotStreamingActor(this::startChildActor); + ThingsPersistenceStreamingActorCreator.startPersistenceStreamingActor(this::startChildActor); final var cleanupConfig = thingsConfig.getThingConfig().getCleanupConfig(); - final var mongoReadJournal = newMongoReadJournal(thingsConfig.getMongoDbConfig(), actorSystem); final Props cleanupActorProps = PersistenceCleanupActor.props(cleanupConfig, mongoReadJournal, CLUSTER_ROLE); startChildActor(PersistenceCleanupActor.ACTOR_NAME, cleanupActorProps); @@ -174,9 +175,10 @@ private static Props getThingSupervisorActorProps(final ActorRef pubSubMediator, final LiveSignalPub liveSignalPub, final ThingPersistenceActorPropsFactory propsFactory, final BlockedNamespaces blockedNamespaces, - final PolicyEnforcerProvider policyEnforcerProvider) { + final PolicyEnforcerProvider policyEnforcerProvider, + final MongoReadJournal mongoReadJournal) { return ThingSupervisorActor.props(pubSubMediator, distributedPubThingEventsForTwin, - liveSignalPub, propsFactory, blockedNamespaces, policyEnforcerProvider); + liveSignalPub, propsFactory, blockedNamespaces, policyEnforcerProvider, mongoReadJournal); } private static MongoReadJournal newMongoReadJournal(final MongoDbConfig mongoDbConfig, diff --git a/things/service/src/main/resources/things.conf b/things/service/src/main/resources/things.conf index dd6dcf944d7..e0741e2ae6c 100755 --- a/things/service/src/main/resources/things.conf +++ b/things/service/src/main/resources/things.conf @@ -54,6 +54,19 @@ ditto { threshold = ${?THING_SNAPSHOT_THRESHOLD} # may be overridden with this environment variable } + event { + # define the DittoHeaders to persist when persisting events to the journal + # those can e.g. be retrieved as additional "audit log" information when accessing a historical thing revision + historical-headers-to-persist = [ + #"ditto-originator" # who (user-subject/connection-pre-auth-subject) issued the event + #"correlation-id" + #"ditto-origin" # which WS session or connection issued the event + #"origin" # the HTTP origin header + #"user-agent" # the HTTP user-agent header + ] + historical-headers-to-persist = ${?THING_EVENT_HISTORICAL_HEADERS_TO_PERSIST} + } + supervisor { exponential-backoff { min = 1s @@ -64,27 +77,62 @@ ditto { } cleanup { + # enabled configures whether background cleanup is enabled or not + # If enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process enabled = true enabled = ${?CLEANUP_ENABLED} + # history-retention-duration configures the duration of how long to "keep" events and snapshots before being + # allowed to remove them in scope of cleanup. + # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read + # journal. + history-retention-duration = 0d + history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} + + # quiet-period defines how long to stay in a state where the background cleanup is not yet started + # Applies after: + # - starting the service + # - each "completed" background cleanup run (all entities were cleaned up) quiet-period = 5m quiet-period = ${?CLEANUP_QUIET_PERIOD} + # interval configures how often a "credit decision" is made. + # The background cleanup works with a credit system and does only generate new "cleanup credits" if the MongoDB + # currently has capacity to do cleanups. interval = 3s interval = ${?CLEANUP_INTERVAL} + # timer-threshold configures the maximum database latency to give out credit for cleanup actions. + # If write operations to the MongoDB within the last `interval` had a `max` value greater to the configured + # threshold, no new cleanup credits will be issued for the next `interval`. + # Which throttles cleanup when MongoDB is currently under heavy (write) load. timer-threshold = 150ms timer-threshold = ${?CLEANUP_TIMER_THRESHOLD} + # credits-per-batch configures how many "cleanup credits" should be generated per `interval` as long as the + # write operations to the MongoDB are less than the configured `timer-threshold`. + # Limits the rate of cleanup actions to this many per credit decision interval. + # One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`. credits-per-batch = 3 credits-per-batch = ${?CLEANUP_CREDITS_PER_BATCH} + # reads-per-query configures the number of snapshots to scan per MongoDB query. + # Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned + # with the amount of "cleanup credits" issued per `interval` - in order to avoid long running queries. reads-per-query = 100 reads-per-query = ${?CLEANUP_READS_PER_QUERY} + # writes-per-credit configures the number of documents to delete for each credit. + # If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead + # to 10 delete operations performed against MongoDB. writes-per-credit = 100 writes-per-credit = ${?CLEANUP_WRITES_PER_CREDIT} + # delete-final-deleted-snapshot configures whether for a deleted entity, the final snapshot (containing the + # "deleted" information) should be deleted or not. + # If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with + # a revision number 1 higher than the previously deleted entity. If the final snapshot is deleted as well, + # recreation of an entity with the same ID will lead to revisionNumber=1 after its recreation. delete-final-deleted-snapshot = false delete-final-deleted-snapshot = ${?CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT} } @@ -229,6 +277,18 @@ akka-contrib-mongodb-persistence-things-journal { } } +akka-contrib-mongodb-persistence-things-journal-read { + class = "akka.contrib.persistence.mongodb.MongoReadJournal" + plugin-dispatcher = "thing-journal-persistence-dispatcher" + + overrides { + journal-collection = "things_journal" + journal-index = "things_journal_index" + realtime-collection = "things_realtime" + metadata-collection = "things_metadata" + } +} + akka-contrib-mongodb-persistence-things-snapshots { class = "akka.contrib.persistence.mongodb.MongoSnapshots" plugin-dispatcher = "thing-snaps-persistence-dispatcher" diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/AbstractThingEnforcementTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/AbstractThingEnforcementTest.java index 5558cc9308a..b658894c2b6 100644 --- a/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/AbstractThingEnforcementTest.java +++ b/things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/AbstractThingEnforcementTest.java @@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit; import org.eclipse.ditto.base.model.auth.AuthorizationSubject; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.internal.utils.pubsub.extractors.AckExtractor; import org.eclipse.ditto.policies.enforcement.PolicyEnforcerProvider; @@ -127,7 +128,8 @@ public > Object wrapForPublicationWithAcks(final S messa new TestSetup.DummyLiveSignalPub(pubSubMediatorProbe.ref()), thingPersistenceActorProbe.ref(), null, - policyEnforcerProvider + policyEnforcerProvider, + Mockito.mock(MongoReadJournal.class) ).withDispatcher("akka.actor.default-dispatcher"), system.guardian(), URLEncoder.encode(THING_ID.toString(), Charset.defaultCharset())); // Actors using "stash()" require the above dispatcher to be configured, otherwise stash() and unstashAll() won't diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/PersistenceActorTestBase.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/PersistenceActorTestBase.java index 7068f4f7eef..fc1473de9a6 100755 --- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/PersistenceActorTestBase.java +++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/PersistenceActorTestBase.java @@ -28,6 +28,7 @@ import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.internal.utils.pubsub.extractors.AckExtractor; import org.eclipse.ditto.internal.utils.pubsubthings.LiveSignalPub; @@ -226,11 +227,13 @@ protected ActorRef createPersistenceActorFor(final ThingId thingId) { protected ActorRef createPersistenceActorWithPubSubFor(final ThingId thingId) { - return actorSystem.actorOf(getPropsOfThingPersistenceActor(thingId, getDistributedPub())); + return actorSystem.actorOf(getPropsOfThingPersistenceActor(thingId, Mockito.mock(MongoReadJournal.class), + getDistributedPub())); } - private Props getPropsOfThingPersistenceActor(final ThingId thingId, final DistributedPub> pub) { - return ThingPersistenceActor.props(thingId, pub, null); + private Props getPropsOfThingPersistenceActor(final ThingId thingId, final MongoReadJournal mongoReadJournal, + final DistributedPub> pub) { + return ThingPersistenceActor.props(thingId, mongoReadJournal, pub, null); } protected ActorRef createSupervisorActorFor(final ThingId thingId) { @@ -258,9 +261,11 @@ public > Object wrapForPublicationWithAcks(final S messa } }, liveSignalPub, - (thingId1, pub, searchShardRegionProxy) -> getPropsOfThingPersistenceActor(thingId1, pub), + (thingId1, mongoReadJournal, pub, searchShardRegionProxy) -> getPropsOfThingPersistenceActor( + thingId1, mongoReadJournal, pub), null, - policyEnforcerProvider); + policyEnforcerProvider, + Mockito.mock(MongoReadJournal.class)); return actorSystem.actorOf(props, thingId.toString()); } diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorSnapshottingTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorSnapshottingTest.java index 0f4d64fed6c..69751dab4c1 100755 --- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorSnapshottingTest.java +++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorSnapshottingTest.java @@ -16,8 +16,10 @@ import java.util.Arrays; import java.util.Collections; +import org.assertj.core.api.Assertions; import org.eclipse.ditto.base.model.common.HttpStatus; import org.eclipse.ditto.base.model.signals.events.EventsourcedEvent; +import org.eclipse.ditto.internal.utils.config.DittoConfigError; import org.eclipse.ditto.internal.utils.test.Retry; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.json.JsonFactory; @@ -233,13 +235,7 @@ public void thingInArbitraryStateIsSnapshotCorrectly() { public void actorCannotBeStartedWithNegativeSnapshotThreshold() { final Config customConfig = createNewDefaultTestConfig().withValue(SNAPSHOT_THRESHOLD, ConfigValueFactory.fromAnyRef(-1)); - setup(customConfig); - - disableLogging(); - new TestKit(actorSystem) {{ - final ActorRef underTest = createPersistenceActorFor(ThingId.of("fail:fail")); - watch(underTest); - expectTerminated(underTest); - }}; + Assertions.assertThatExceptionOfType(DittoConfigError.class) + .isThrownBy(() -> setup(customConfig)); } } diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorTest.java index 533a758568d..e87263621fb 100755 --- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorTest.java +++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceActorTest.java @@ -44,6 +44,7 @@ import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.events.Event; import org.eclipse.ditto.internal.utils.cluster.DistPubSubAccess; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.test.Retry; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.json.JsonFactory; @@ -116,6 +117,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestWatcher; +import org.mockito.Mockito; import org.slf4j.LoggerFactory; import com.typesafe.config.Config; @@ -304,7 +306,8 @@ public void tryToCreateThingWithDifferentThingId() { final Thing thing = createThingV2WithRandomId(); final CreateThing createThing = CreateThing.of(thing, null, dittoHeadersV2); - final Props props = ThingPersistenceActor.props(thingIdOfActor, getDistributedPub(), null); + final Props props = ThingPersistenceActor.props(thingIdOfActor, Mockito.mock(MongoReadJournal.class), + getDistributedPub(), null); final TestActorRef underTest = TestActorRef.create(actorSystem, props); final ThingPersistenceActor thingPersistenceActor = underTest.underlyingActor(); final PartialFunction receiveCommand = thingPersistenceActor.receiveCommand(); diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceOperationsActorIT.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceOperationsActorIT.java index f414733180a..5316bb94124 100644 --- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceOperationsActorIT.java +++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/actors/ThingPersistenceOperationsActorIT.java @@ -15,6 +15,7 @@ import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.internal.utils.persistence.mongo.ops.eventsource.MongoEventSourceITAssertions; +import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; import org.eclipse.ditto.internal.utils.pubsub.DistributedPub; import org.eclipse.ditto.internal.utils.pubsub.extractors.AckExtractor; import org.eclipse.ditto.internal.utils.pubsubthings.LiveSignalPub; @@ -149,10 +150,15 @@ public > Object wrapForPublicationWithAcks(final S messa } }, liveSignalPub, - (thingId, distributedPub, searchShardRegionProxy) -> ThingPersistenceActor.props(thingId, - distributedPub, null), + (thingId, mongoReadJournal, distributedPub, searchShardRegionProxy) -> ThingPersistenceActor.props( + thingId, + mongoReadJournal, + distributedPub, + null + ), null, - policyEnforcerProvider); + policyEnforcerProvider, + Mockito.mock(MongoReadJournal.class)); return system.actorOf(props, id.toString()); } diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapterTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapterTest.java index a6b097f5de1..062f530f58f 100644 --- a/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapterTest.java +++ b/things/service/src/test/java/org/eclipse/ditto/things/service/persistence/serializer/ThingMongoEventAdapterTest.java @@ -17,6 +17,7 @@ import java.time.Instant; import java.util.List; +import java.util.Set; import org.bson.BsonDocument; import org.eclipse.ditto.base.model.headers.DittoHeaders; @@ -30,7 +31,12 @@ import org.eclipse.ditto.things.model.signals.events.ThingDeleted; import org.junit.Test; +import com.typesafe.config.ConfigFactory; + +import akka.actor.ActorSystem; +import akka.actor.ExtendedActorSystem; import akka.persistence.journal.EventSeq; +import akka.persistence.journal.Tagged; import scala.jdk.javaapi.CollectionConverters; /** @@ -41,7 +47,8 @@ public final class ThingMongoEventAdapterTest { private final ThingMongoEventAdapter underTest; public ThingMongoEventAdapterTest() { - underTest = new ThingMongoEventAdapter(null); + underTest = new ThingMongoEventAdapter( + (ExtendedActorSystem) ActorSystem.create("test", ConfigFactory.load("test"))); } @Test @@ -93,11 +100,13 @@ public void toJournalReturnsBsonDocument() { " \"attributes\" : {\n" + " \"hello\" : \"cloud\"\n" + " }\n" + - " }\n" + + " },\n" + + " \"__hh\": {}\n" + " }"; final BsonDocument bsonEvent = BsonDocument.parse(journalEntry); + final Tagged tagged = new Tagged(bsonEvent, Set.of()); - assertThat(underTest.toJournal(thingCreated)).isEqualTo(bsonEvent); + assertThat(underTest.toJournal(thingCreated)).isEqualTo(tagged); } @Test diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalCommandRegistryTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalCommandRegistryTest.java index 8aea6bf6ce3..0941014ce8a 100644 --- a/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalCommandRegistryTest.java +++ b/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalCommandRegistryTest.java @@ -17,6 +17,7 @@ import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; import org.eclipse.ditto.internal.models.streaming.SudoStreamPids; import org.eclipse.ditto.internal.utils.health.RetrieveHealth; @@ -55,6 +56,8 @@ public ThingsServiceGlobalCommandRegistryTest() { PurgeEntities.class, ModifySplitBrainResolver.class, PublishSignal.class, + SubscribeForPersistedEvents.class, + PublishSignal.class, CreateSubscription.class, QueryThings.class, SudoCountThings.class diff --git a/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalEventRegistryTest.java b/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalEventRegistryTest.java index f466d1b1c06..91c529610b0 100644 --- a/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalEventRegistryTest.java +++ b/things/service/src/test/java/org/eclipse/ditto/things/service/starter/ThingsServiceGlobalEventRegistryTest.java @@ -12,10 +12,11 @@ */ package org.eclipse.ditto.things.service.starter; -import org.eclipse.ditto.policies.model.signals.events.PolicyModified; -import org.eclipse.ditto.things.api.ThingSnapshotTaken; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; import org.eclipse.ditto.internal.utils.persistentactors.EmptyEvent; import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases; +import org.eclipse.ditto.policies.model.signals.events.PolicyModified; +import org.eclipse.ditto.things.api.ThingSnapshotTaken; import org.eclipse.ditto.things.model.signals.events.FeatureDeleted; import org.eclipse.ditto.thingsearch.api.events.ThingsOutOfSync; import org.eclipse.ditto.thingsearch.model.signals.events.SubscriptionCreated; @@ -28,6 +29,8 @@ public ThingsServiceGlobalEventRegistryTest() { ThingSnapshotTaken.class, EmptyEvent.class, PolicyModified.class, + StreamingSubscriptionComplete.class, + PolicyModified.class, SubscriptionCreated.class, ThingsOutOfSync.class ); diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java index ff8cdc81f97..512235581bc 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java @@ -17,6 +17,7 @@ import org.eclipse.ditto.base.api.devops.signals.commands.ExecutePiggybackCommand; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; +import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection; @@ -60,7 +61,8 @@ public ThingsSearchServiceGlobalCommandRegistryTest() { CleanupPersistence.class, ModifyConnection.class, ModifySplitBrainResolver.class, - RetrieveConnection.class + RetrieveConnection.class, + SubscribeForPersistedEvents.class ); } diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalEventRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalEventRegistryTest.java index 1c89950ff16..b0913cf70dc 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalEventRegistryTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalEventRegistryTest.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.thingsearch.service.starter; +import org.eclipse.ditto.base.model.signals.events.streaming.StreamingSubscriptionComplete; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionModified; import org.eclipse.ditto.internal.utils.test.GlobalEventRegistryTestCases; import org.eclipse.ditto.policies.model.signals.events.ResourceDeleted; @@ -29,7 +30,8 @@ public ThingsSearchServiceGlobalEventRegistryTest() { SubscriptionCreated.class, ThingsOutOfSync.class, ThingSnapshotTaken.class, - ConnectionModified.class + ConnectionModified.class, + StreamingSubscriptionComplete.class ); } From a4c0634319c3e6ed030ac23f89a169cd2b61228f Mon Sep 17 00:00:00 2001 From: thfries Date: Fri, 6 Jan 2023 14:44:39 +0100 Subject: [PATCH 006/173] Explorer UI - SSE and optimistic locking - custom web component for CRUD toolbar - API allows to return headers if needed - adapt editors with new toolbar and ETag handling - fixed modified date and revision from SSE - avoid feature editors jumping because of badge in header - fixes some static type checks in utils --- ui/index.css | 5 + ui/index.html | 39 ++++- ui/modules/api.js | 9 +- ui/modules/things/attributes.js | 140 +++++++++++----- ui/modules/things/featureMessages.js | 16 +- ui/modules/things/features.html | 25 +-- ui/modules/things/features.js | 230 +++++++++++++++++---------- ui/modules/things/searchFilter.js | 2 +- ui/modules/things/things.html | 52 ++---- ui/modules/things/thingsCRUD.js | 115 +++++++------- ui/modules/things/thingsSSE.js | 3 +- ui/modules/things/thingsSearch.js | 2 +- ui/modules/utils.js | 23 +-- ui/modules/utils/crudToolbar.js | 95 +++++++++++ 14 files changed, 488 insertions(+), 268 deletions(-) create mode 100644 ui/modules/utils/crudToolbar.js diff --git a/ui/index.css b/ui/index.css index 8fae4bf6f84..401a2c5ab0c 100644 --- a/ui/index.css +++ b/ui/index.css @@ -175,3 +175,8 @@ hr { max-height: 80vh; overflow-y: auto; } + +h5>.badge { + vertical-align: top; + font-size: 0.6em; +} \ No newline at end of file diff --git a/ui/index.html b/ui/index.html index cc3d1f59465..83ba6a5a44b 100644 --- a/ui/index.html +++ b/ui/index.html @@ -45,6 +45,7 @@ integrity="sha512-GZ1RIgZaSc8rnco/8CXfRdCpDxRCphenIiZ2ztLy3XQfCbQUSCuk8IudvNHxkRA3oUg6q0qejgN/qqyG1duv5Q==" crossorigin="anonymous" type="text/javascript" charset="utf-8"> + @@ -107,7 +108,7 @@ -

+
@@ -128,8 +129,42 @@
-
+
+ + \ No newline at end of file diff --git a/ui/modules/api.js b/ui/modules/api.js index a512d2c746e..bc639d0f08f 100644 --- a/ui/modules/api.js +++ b/ui/modules/api.js @@ -298,9 +298,10 @@ export function setAuthHeader(forDevOps) { * @param {String} path of the Ditto call (e.g. '/things') * @param {Object} body payload for the api call * @param {Object} additionalHeaders object with additional header fields + * @param {boolean} returnHeaders request full response instead of json content * @return {Object} result as json object */ -export async function callDittoREST(method, path, body, additionalHeaders) { +export async function callDittoREST(method, path, body, additionalHeaders, returnHeaders = false) { let response; try { response = await fetch(Environments.current().api_uri + '/api/2' + path, { @@ -327,7 +328,11 @@ export async function callDittoREST(method, path, body, additionalHeaders) { throw new Error('An error occurred: ' + response.status); } if (response.status !== 204) { - return response.json(); + if (returnHeaders) { + return response; + } else { + return response.json(); + } } else { return null; } diff --git a/ui/modules/things/attributes.js b/ui/modules/things/attributes.js index 13f81087960..1495193623f 100644 --- a/ui/modules/things/attributes.js +++ b/ui/modules/things/attributes.js @@ -1,3 +1,4 @@ +/* eslint-disable require-jsdoc */ /* * Copyright (c) 2022 Contributors to the Eclipse Foundation * @@ -16,74 +17,115 @@ import * as Utils from '../utils.js'; import * as Things from './things.js'; const dom = { - attributesTable: null, - attributePath: null, - attributeValue: null, - attributeCount: null, - putAttribute: null, - deleteAttribute: null, + tbodyAttributes: null, + crudAttribute: null, + inputAttributeValue: null, + badgeAttributeCount: null, }; +let eTag; + /** * Initializes components. Should be called after DOMContentLoaded event */ export function ready() { + Things.addChangeListener(onThingChanged); + Utils.getAllElementsById(dom); - dom.attributesTable.onclick = (event) => { - if (event.target && event.target.nodeName === 'TD') { - const path = event.target.parentNode.children[0].innerText; - dom.attributePath.value = path; - dom.attributeValue.value = attributeToString(Things.theThing.attributes[path]); - } - }; + dom.tbodyAttributes.onclick = onAttributeTableClick; + + dom.crudAttribute.addEventListener('onCreateClick', onCreateAttributeClick); + dom.crudAttribute.addEventListener('onUpdateClick', onUpdateAttributeClick); + dom.crudAttribute.addEventListener('onDeleteClick', onDeleteAttributeClick); + dom.crudAttribute.addEventListener('onEditToggle', onEditToggle); +} + +function onCreateAttributeClick() { + Utils.assert(dom.crudAttribute.idValue, 'Attribute path must not be empty', dom.crudAttribute.validationElement); + Utils.assert(!Things.theThing['attributes'] || !Object.keys(Things.theThing.attributes).includes(dom.crudAttribute.idValue), + `Attribute path ${dom.crudAttribute.idValue} already exists in Thing`, + dom.crudAttribute.validationElement); + Utils.assert(dom.inputAttributeValue.value, 'Attribute value must not be empty', dom.inputAttributeValue); - dom.putAttribute.onclick = clickAttribute('PUT'); - dom.deleteAttribute.onclick = clickAttribute('DELETE'); + updateAttribute('PUT'); +} +function onUpdateAttributeClick() { + Utils.assert(dom.inputAttributeValue.value, 'Attribute value must not be empty'); + updateAttribute('PUT'); +} + +function onDeleteAttributeClick() { + Utils.confirm(`Are you sure you want to delete attribute
'${dom.crudAttribute.idValue}'?`, 'Delete', () => { + updateAttribute('DELETE'); + }); +} - Things.addChangeListener(updateAttributesTable); +function onAttributeTableClick(event) { + if (event.target && event.target.nodeName === 'TD') { + const path = event.target.parentNode.children[0].innerText; + if (dom.crudAttribute.idValue === path) { + refreshAttribute(null); + } else { + refreshAttribute(Things.theThing, path); + } + } } /** * Creates a onclick handler function * @param {String} method PUT or DELETE - * @return {function} Click handler function to PUT or DELETE Ditto attribute */ -function clickAttribute(method) { - return function() { - Utils.assert(Things.theThing, 'No Thing selected'); - Utils.assert(dom.attributePath.value, 'Attribute path is empty'); - Utils.assert(method !== 'PUT' || dom.attributeValue.value, 'Attribute value is empty'); - API.callDittoREST( - method, - `/things/${Things.theThing.thingId}/attributes/${dom.attributePath.value}`, - method === 'PUT' ? attributeFromString(dom.attributeValue.value) : null, - ).then(() => Things.refreshThing(Things.theThing.thingId)); - }; +function updateAttribute(method) { + API.callDittoREST( + method, + `/things/${Things.theThing.thingId}/attributes/${dom.crudAttribute.idValue}`, + method === 'PUT' ? attributeFromString(dom.inputAttributeValue.value) : null, + { + 'if-match': method === 'PUT' ? eTag : '*', + }, + ).then(() => { + if (method === 'PUT') { + dom.crudAttribute.toggleEdit(); + } + Things.refreshThing(Things.theThing.thingId); + }); } -/** - * Updates UI compoents for attributes - */ -function updateAttributesTable() { - dom.attributesTable.innerHTML = ''; +function refreshAttribute(thing, attributePath) { + if (dom.crudAttribute.isEditing) { + return; + } + + if (thing) { + dom.crudAttribute.idValue = attributePath; + dom.inputAttributeValue.value = attributeToString(thing.attributes[attributePath]); + } else { + dom.crudAttribute.idValue = null; + dom.inputAttributeValue.value = null; + } +} + +function onThingChanged(thing) { + dom.crudAttribute.editDisabled = (thing === null); + + dom.tbodyAttributes.innerHTML = ''; let count = 0; let thingHasAttribute = false; - if (Things.theThing && Things.theThing.attributes) { + if (thing && thing.attributes) { Object.keys(Things.theThing.attributes).forEach((path) => { - if (path === dom.attributePath.value) { - dom.attributeValue.value = attributeToString(Things.theThing.attributes[path]); + if (path === dom.crudAttribute.idValue) { + refreshAttribute(Things.theThing, path); thingHasAttribute = true; } - Utils.addTableRow(dom.attributesTable, path, path === dom.attributePath.value, false, + Utils.addTableRow(dom.tbodyAttributes, path, path === dom.crudAttribute.idValue, false, attributeToString(Things.theThing.attributes[path])); count++; }); } - dom.attributeCount.innerText = count > 0 ? count : ''; + dom.badgeAttributeCount.innerText = count > 0 ? count : ''; if (!thingHasAttribute) { - dom.attributePath.value = null; - dom.attributeValue.value = null; + refreshAttribute(null); } } @@ -110,3 +152,21 @@ function attributeFromString(attribute) { return attribute; } } + +function onEditToggle(event) { + if (event.detail) { + API.callDittoREST('GET', `/things/${Things.theThing.thingId}/attributes/${dom.crudAttribute.idValue}`, + null, null, true) + .then((response) => { + eTag = response.headers.get('ETag'); + return response.json(); + }) + .then((attributeValue) => { + dom.inputAttributeValue.disabled = false; + dom.inputAttributeValue.value = attributeToString(attributeValue); + }); + } else { + dom.inputAttributeValue.disabled = true; + dom.inputAttributeValue.value = attributeToString(Things.theThing.attributes[dom.crudAttribute.idValue]); + } +} diff --git a/ui/modules/things/featureMessages.js b/ui/modules/things/featureMessages.js index 52172bfe717..69f1cdd24f0 100644 --- a/ui/modules/things/featureMessages.js +++ b/ui/modules/things/featureMessages.js @@ -18,6 +18,8 @@ import * as Utils from '../utils.js'; import * as Things from './things.js'; import * as Features from './features.js'; +let theFeatureId; + const dom = { inputMessageSubject: null, inputMessageTimeout: null, @@ -26,7 +28,6 @@ const dom = { buttonMessageFavourite: null, ulMessageTemplates: null, favIconMessage: null, - theFeatureId: null, tableValidationFeature: null, }; @@ -54,7 +55,7 @@ export async function ready() { dom.buttonMessageSend.onclick = () => { - Utils.assert(dom.theFeatureId.value, 'Please select a Feature', dom.tableValidationFeature); + Utils.assert(theFeatureId, 'Please select a Feature', dom.tableValidationFeature); Utils.assert(dom.inputMessageSubject.value, 'Please give a Subject', dom.inputMessageSubject); Utils.assert(dom.inputMessageTimeout.value, 'Please give a timeout', dom.inputMessageTimeout); messageFeature(); @@ -62,7 +63,7 @@ export async function ready() { dom.buttonMessageFavourite.onclick = () => { const templateName = dom.inputMessageTemplate.value; - const featureId = dom.theFeatureId.value; + const featureId = theFeatureId; Utils.assert(featureId, 'Please select a Feature', dom.tableValidationFeature); Utils.assert(templateName, 'Please give a name for the template', dom.inputMessageTemplate); Environments.current().messageTemplates[featureId] = Environments.current().messageTemplates[featureId] || {}; @@ -85,7 +86,7 @@ export async function ready() { dom.ulMessageTemplates.addEventListener('click', (event) => { if (event.target && event.target.classList.contains('dropdown-item')) { dom.favIconMessage.classList.replace('bi-star', 'bi-star-fill'); - const template = Environments.current().messageTemplates[dom.theFeatureId.value][event.target.textContent]; + const template = Environments.current().messageTemplates[theFeatureId][event.target.textContent]; dom.inputMessageTemplate.value = event.target.textContent; dom.inputMessageSubject.value = template.subject; dom.inputMessageTimeout.value = template.timeout; @@ -114,7 +115,7 @@ function messageFeature() { const payload = JSON.parse(acePayload.getValue()); aceResponse.setValue(''); API.callDittoREST('POST', '/things/' + Things.theThing.thingId + - '/features/' + dom.theFeatureId.value + + '/features/' + theFeatureId + '/inbox/messages/' + dom.inputMessageSubject.value + '?timeout=' + dom.inputMessageTimeout.value, payload, @@ -151,15 +152,16 @@ function clearAllFields() { function refillTemplates() { dom.ulMessageTemplates.innerHTML = ''; Utils.addDropDownEntries(dom.ulMessageTemplates, ['Saved message templates'], true); - if (dom.theFeatureId.value && Environments.current().messageTemplates[dom.theFeatureId.value]) { + if (theFeatureId && Environments.current().messageTemplates[theFeatureId]) { Utils.addDropDownEntries( dom.ulMessageTemplates, - Object.keys(Environments.current().messageTemplates[dom.theFeatureId.value]), + Object.keys(Environments.current().messageTemplates[theFeatureId]), ); } } function onFeatureChanged(featureId) { + theFeatureId = featureId; clearAllFields(); refillTemplates(); } diff --git a/ui/modules/things/features.html b/ui/modules/things/features.html index 57938353c40..2278dff74b1 100644 --- a/ui/modules/things/features.html +++ b/ui/modules/things/features.html @@ -11,7 +11,7 @@ ~ SPDX-License-Identifier: EPL-2.0 -->
- Features
+ Features
@@ -22,7 +22,7 @@
- +
@@ -34,25 +34,12 @@
-
-
- - - - - -
+
- +
-
+
@@ -61,7 +48,7 @@
-
+
diff --git a/ui/modules/things/features.js b/ui/modules/things/features.js index f4c304d9560..e67fcf282f2 100644 --- a/ui/modules/things/features.js +++ b/ui/modules/things/features.js @@ -27,17 +27,19 @@ export function addChangeListener(observer) { } function notifyAll() { - observers.forEach((observer) => observer.call(null, dom.theFeatureId.value)); + observers.forEach((observer) => observer.call(null, dom.crudFeature.idValue)); } let featurePropertiesEditor; let featureDesiredPropertiesEditor; +let eTag; + const dom = { - theFeatureId: null, - featureDefinition: null, - featureCount: null, - featuresTable: null, + crudFeature: null, + inputFeatureDefinition: null, + badgeFeatureCount: null, + tbodyFeatures: null, tableValidationFeature: null, }; @@ -45,77 +47,76 @@ const dom = { * Initializes components. Should be called after DOMContentLoaded event */ export function ready() { - Utils.getAllElementsById(dom); - - Utils.addValidatorToTable(dom.featuresTable, dom.tableValidationFeature); - - - dom.featuresTable.onclick = (event) => { - if (event.target && event.target.nodeName === 'TD') { - dom.theFeatureId.value = event.target.textContent; - refreshFeature(Things.theThing, dom.theFeatureId.value); - } - }; + Things.addChangeListener(onThingChanged); - document.getElementById('createFeature').onclick = () => { - if (!dom.theFeatureId.value) { - dom.theFeatureId.value = 'new-feature'; - } - createFeature(dom.theFeatureId.value); - }; + Utils.getAllElementsById(dom); - document.getElementById('putFeature').onclick = () => updateFeature('PUT'); - document.getElementById('deleteFeature').onclick = () => updateFeature('DELETE'); + Utils.addValidatorToTable(dom.tbodyFeatures, dom.tableValidationFeature); - featurePropertiesEditor = ace.edit('featurePropertiesEditor'); - featureDesiredPropertiesEditor = ace.edit('featureDesiredPropertiesEditor'); + dom.tbodyFeatures.onclick = onFeaturesTableClick; - featurePropertiesEditor.session.setMode('ace/mode/json'); - featureDesiredPropertiesEditor.session.setMode('ace/mode/json'); + dom.crudFeature.editDisabled = true; + dom.crudFeature.addEventListener('onCreateClick', onCreateFeatureClick); + dom.crudFeature.addEventListener('onUpdateClick', onUpdateFeatureClick); + dom.crudFeature.addEventListener('onDeleteClick', onDeleteFeatureClick); + dom.crudFeature.addEventListener('onEditToggle', onEditToggle); - featurePropertiesEditor.on('dblclick', (event) => { - if (!event.domEvent.shiftKey) { - return; - } + featurePropertiesEditor = Utils.createAceEditor('featurePropertiesEditor', 'ace/mode/json', true); + featureDesiredPropertiesEditor = Utils.createAceEditor('featureDesiredPropertiesEditor', 'ace/mode/json', true); - setTimeout(() => { - const token = featurePropertiesEditor.getSelectedText(); - if (token) { - const path = '$..' + token.replace(/['"]+/g, '').trim(); - const res = JSONPath({ - json: JSON.parse(featurePropertiesEditor.getValue()), - path: path, - resultType: 'pointer', - }); - Fields.proposeNewField('features/' + dom.theFeatureId.value + '/properties' + res); - } - }, 10); - }); + featurePropertiesEditor.on('dblclick', onFeaturePropertiesDblClick); document.querySelector('a[data-bs-target="#tabCrudFeature"]').addEventListener('shown.bs.tab', (event) => { featurePropertiesEditor.renderer.updateFull(); featureDesiredPropertiesEditor.renderer.updateFull(); }); +} - Things.addChangeListener(onThingChanged); +function onUpdateFeatureClick() { + updateFeature('PUT'); } -/** - * Creates a new empty feature for the given thing in Ditto - * @param {String} newFeatureId Name of the new feature. - */ -function createFeature(newFeatureId) { - console.assert(newFeatureId && newFeatureId != '', 'newFeatureId expected'); - Utils.assert(Things.theThing, 'No Thing selected'); - if (Things.theThing['features']) { - Utils.assert(!Object.keys(Things.theThing.features).includes(newFeatureId), - `Feature ID ${newFeatureId} already exists in Thing`); +function onDeleteFeatureClick() { + Utils.confirm(`Are you sure you want to delete feature
'${dom.crudFeature.idValue}'?`, 'Delete', () => { + updateFeature('DELETE'); + }); +} + +function onFeaturePropertiesDblClick(event) { + if (!event.domEvent.shiftKey) { + return; } - API.callDittoREST('PUT', - '/things/' + Things.theThing.thingId + '/features/' + newFeatureId, - {}, - ).then(() => Things.refreshThing(Things.theThing.thingId)); + setTimeout(() => { + const token = featurePropertiesEditor.getSelectedText(); + if (token) { + const path = '$..' + token.replace(/['"]+/g, '').trim(); + const res = JSONPath({ + json: JSON.parse(featurePropertiesEditor.getValue()), + path: path, + resultType: 'pointer', + }); + Fields.proposeNewField('features/' + dom.crudFeature.idValue + '/properties' + res); + } + }, 10); +} + +function onCreateFeatureClick() { + Utils.assert(dom.crudFeature.idValue, 'Feature ID must not be empty', dom.crudFeature.validationElement); + Utils.assert(!Things.theThing['features'] || !Object.keys(Things.theThing.features).includes(dom.crudFeature.idValue), + `Feature ID ${dom.crudFeature.idValue} already exists in Thing`, + dom.crudFeature.validationElement); + updateFeature('PUT'); +} + +function onFeaturesTableClick(event) { + if (event.target && event.target.nodeName === 'TD') { + if (dom.crudFeature.idValue === event.target.textContent) { + refreshFeature(null); + } else { + refreshFeature(Things.theThing, event.target.textContent); + } + } } /** @@ -124,13 +125,13 @@ function createFeature(newFeatureId) { */ function updateFeature(method) { Utils.assert(Things.theThing, 'No Thing selected'); - Utils.assert(dom.theFeatureId.value, 'No Feature selected'); + Utils.assert(dom.crudFeature.idValue, 'No Feature selected'); const featureObject = {}; const featureProperties = featurePropertiesEditor.getValue(); const featureDesiredProperties = featureDesiredPropertiesEditor.getValue(); - if (dom.featureDefinition.value) { - featureObject.definition = dom.featureDefinition.value.split(','); + if (dom.inputFeatureDefinition.value) { + featureObject.definition = dom.inputFeatureDefinition.value.split(','); } if (featureProperties) { featureObject.properties = JSON.parse(featureProperties); @@ -141,40 +142,57 @@ function updateFeature(method) { API.callDittoREST( method, - '/things/' + Things.theThing.thingId + '/features/' + dom.theFeatureId.value, + '/things/' + Things.theThing.thingId + '/features/' + dom.crudFeature.idValue, method === 'PUT' ? featureObject : null, - ).then(() => Things.refreshThing(Things.theThing.thingId) - ).catch( + { + 'if-match': method === 'PUT' ? eTag : '*' + } + ).then(() => { + if (method === 'PUT') { + dom.crudFeature.toggleEdit(); + } + Things.refreshThing(Things.theThing.thingId); + }).catch( // nothing to clean-up if featureUpdate failed ); } -/** - * Initializes all UI components for the given single feature of the given thing, if no thing is given the UI is cleared - * @param {Object} thing thing the feature values are taken from - * @param {String} feature FeatureId to be refreshed - */ -function refreshFeature(thing, feature) { - if (thing) { - dom.theFeatureId.value = feature; - dom.featureDefinition.value = thing.features[feature]['definition'] ? thing.features[feature].definition : null; - if (thing.features[feature]['properties']) { - featurePropertiesEditor.setValue(JSON.stringify(thing.features[feature].properties, null, 4), -1); +function updateFeatureEditors(featureJson) { + if (featureJson) { + dom.inputFeatureDefinition.value = featureJson['definition'] ? featureJson.definition : null; + if (featureJson['properties']) { + featurePropertiesEditor.setValue(JSON.stringify(featureJson.properties, null, 4), -1); } else { featurePropertiesEditor.setValue(''); } - if (thing.features[feature]['desiredProperties']) { - featureDesiredPropertiesEditor.setValue(JSON.stringify(thing.features[feature].desiredProperties, null, 4), -1); + if (featureJson['desiredProperties']) { + featureDesiredPropertiesEditor.setValue(JSON.stringify(featureJson.desiredProperties, null, 4), -1); } else { featureDesiredPropertiesEditor.setValue(''); } } else { - dom.theFeatureId.value = null; - dom.featureDefinition.value = null; + dom.inputFeatureDefinition.value = null; featurePropertiesEditor.setValue(''); featureDesiredPropertiesEditor.setValue(''); } - notifyAll(); +} + +/** + * Initializes all UI components for the given single feature of the given thing, if no thing is given the UI is cleared + * @param {Object} thing thing the feature values are taken from + * @param {String} featureId FeatureId to be refreshed + */ +function refreshFeature(thing, featureId) { + if (!dom.crudFeature.isEditing) { + if (thing) { + dom.crudFeature.idValue = featureId; + updateFeatureEditors(thing.features[featureId]); + } else { + dom.crudFeature.idValue = null; + updateFeatureEditors(null); + } + notifyAll(); + } } /** @@ -182,23 +200,59 @@ function refreshFeature(thing, feature) { * @param {Object} thing UI is initialized for the features of the given thing */ function onThingChanged(thing) { + dom.crudFeature.editDisabled = (thing === null); // Update features table - dom.featuresTable.innerHTML = ''; + dom.tbodyFeatures.innerHTML = ''; let count = 0; let thingHasFeature = false; if (thing && thing.features) { for (const key of Object.keys(thing.features)) { - if (key === dom.theFeatureId.value) { + if (key === dom.crudFeature.idValue) { refreshFeature(thing, key); thingHasFeature = true; } - Utils.addTableRow(dom.featuresTable, key, key === dom.theFeatureId.value); + Utils.addTableRow(dom.tbodyFeatures, key, key === dom.crudFeature.idValue); count++; } } - dom.featureCount.textContent = count > 0 ? count : ''; + dom.badgeFeatureCount.textContent = count > 0 ? count : ''; if (!thingHasFeature) { - dom.theFeatureId.value = null; - refreshFeature(); + dom.crudFeature.idValue = null; + updateFeatureEditors(null); + } +} + +function onEditToggle(event) { + if (event.detail) { + API.callDittoREST('GET', `/things/${Things.theThing.thingId}/features/${dom.crudFeature.idValue}`, null, null, true) + .then((response) => { + eTag = response.headers.get('ETag'); + return response.json(); + }) + .then((featureJson) => { + enableDisableEditors(); + updateFeatureEditors(featureJson); + }); + } else { + enableDisableEditors(); + clearEditorsAfterCancel(); + dom.crudFeature.validationElement.classList.remove('is-invalid'); + } + + function enableDisableEditors() { + dom.inputFeatureDefinition.disabled = !event.detail; + featurePropertiesEditor.setReadOnly(!event.detail); + featurePropertiesEditor.renderer.setShowGutter(event.detail); + featureDesiredPropertiesEditor.setReadOnly(!event.detail); + featureDesiredPropertiesEditor.renderer.setShowGutter(event.detail); + } + + function clearEditorsAfterCancel() { + if (dom.crudFeature.idValue) { + refreshFeature(Things.theThing, dom.crudFeature.idValue); + } else { + refreshFeature(null); + } } } + diff --git a/ui/modules/things/searchFilter.js b/ui/modules/things/searchFilter.js index c0a97d54e6b..fbbbe2f72ef 100644 --- a/ui/modules/things/searchFilter.js +++ b/ui/modules/things/searchFilter.js @@ -60,7 +60,7 @@ export async function ready() { checkIfFavourite(); const filterEditNeeded = checkAndMarkParameter(); if (!filterEditNeeded) { - ThingsSearch.searchThings(event.target.textContent); + ThingsSearch.searchTriggered(event.target.textContent); } } }); diff --git a/ui/modules/things/things.html b/ui/modules/things/things.html index 67a85438ee4..718866f7761 100644 --- a/ui/modules/things/things.html +++ b/ui/modules/things/things.html @@ -67,33 +67,14 @@
Things
- +
-
-
- - - - - - -
-
+
- +
@@ -104,41 +85,38 @@
Things
-
+
-
Attributes
+
Attributes

- +
-
- - - - -
-
- - -
+ +
+ + +
+
+
diff --git a/ui/modules/things/thingsCRUD.js b/ui/modules/things/thingsCRUD.js index 90e613ef323..115ca1294d2 100644 --- a/ui/modules/things/thingsCRUD.js +++ b/ui/modules/things/thingsCRUD.js @@ -20,20 +20,13 @@ import * as Things from './things.js'; let thingJsonEditor; -let isEditing = false; - let thingTemplates; +let eTag; + const dom = { - thingDetails: null, - thingId: null, - modalThingsEdit: null, - iThingsEdit: null, - divThingsCRUD: null, - buttonEditThing: null, - buttonCreateThing: null, - buttonSaveThing: null, - buttonDeleteThing: null, + tbodyThingDetails: null, + crudThings: null, buttonThingDefinitions: null, inputThingDefinition: null, ulThingDefinitions: null, @@ -52,10 +45,10 @@ export async function ready() { loadThingTemplates(); dom.ulThingDefinitions.addEventListener('click', onThingDefinitionsClick); - dom.buttonCreateThing.onclick = onCreateThingClick; - dom.buttonSaveThing.onclick = onSaveThingClick; - dom.buttonDeleteThing.onclick = onDeleteThingClick; - dom.buttonEditThing.onclick = toggleEdit; + dom.crudThings.addEventListener('onCreateClick', onCreateThingClick); + dom.crudThings.addEventListener('onUpdateClick', onUpdateThingClick); + dom.crudThings.addEventListener('onDeleteClick', onDeleteThingClick); + dom.crudThings.addEventListener('onEditToggle', onEditToggle); document.querySelector('a[data-bs-target="#tabModifyThing"]').addEventListener('shown.bs.tab', (event) => { thingJsonEditor.renderer.updateFull(); @@ -63,9 +56,8 @@ export async function ready() { } function onDeleteThingClick() { - Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId); - Utils.confirm(`Are you sure you want to delete thing
'${dom.thingId.value}'?`, 'Delete', () => { - API.callDittoREST('DELETE', `/things/${dom.thingId.value}`, null, + Utils.confirm(`Are you sure you want to delete thing
'${dom.crudThings.idValue}'?`, 'Delete', () => { + API.callDittoREST('DELETE', `/things/${dom.crudThings.idValue}`, null, { 'if-match': '*', }, @@ -75,29 +67,28 @@ function onDeleteThingClick() { }); } -function onSaveThingClick() { - Utils.assert(dom.thingId.value, 'Thing ID is empty', dom.thingId); - API.callDittoREST('PUT', `/things/${dom.thingId.value}`, JSON.parse(thingJsonEditor.getValue()), +function onUpdateThingClick() { + API.callDittoREST('PUT', `/things/${dom.crudThings.idValue}`, JSON.parse(thingJsonEditor.getValue()), { - 'if-match': '*', + 'if-match': eTag, }, ).then(() => { - toggleEdit(); - Things.refreshThing(dom.thingId.value, null); + dom.crudThings.toggleEdit(); + Things.refreshThing(dom.crudThings.idValue, null); }); } async function onCreateThingClick() { const editorValue = thingJsonEditor.getValue(); - if (dom.thingId.value !== undefined && dom.thingId.value !== '') { + if (dom.crudThings.idValue !== undefined && dom.crudThings.idValue !== '') { API.callDittoREST('PUT', - '/things/' + dom.thingId.value, + '/things/' + dom.crudThings.idValue, editorValue === '' ? {} : JSON.parse(editorValue), { 'if-none-match': '*', }, ).then((data) => { - toggleEdit(); + dom.crudThings.toggleEdit(); Things.refreshThing(data.thingId, () => { ThingsSearch.getThings([data.thingId]); }); @@ -105,7 +96,7 @@ async function onCreateThingClick() { } else { API.callDittoREST('POST', '/things', editorValue === '' ? {} : JSON.parse(editorValue)) .then((data) => { - toggleEdit(); + dom.crudThings.toggleEdit(); Things.refreshThing(data.thingId, () => { ThingsSearch.getThings([data.thingId]); }); @@ -115,7 +106,7 @@ async function onCreateThingClick() { function onThingDefinitionsClick(event) { Things.setTheThing(null); - isEditing = true; + // isEditing = true; dom.inputThingDefinition.value = event.target.textContent; thingJsonEditor.setValue(JSON.stringify(thingTemplates[event.target.textContent], null, 2), -1); } @@ -135,7 +126,7 @@ function loadThingTemplates() { * @param {Object} thingJson Thing json */ function onThingChanged(thingJson) { - if (isEditing) { + if (dom.crudThings.isEditing) { return; } @@ -143,21 +134,20 @@ function onThingChanged(thingJson) { updateThingJsonEditor(); function updateThingDetailsTable() { - dom.thingDetails.innerHTML = ''; + dom.tbodyThingDetails.innerHTML = ''; if (thingJson) { - Utils.addTableRow(dom.thingDetails, 'thingId', false, true, thingJson.thingId); - Utils.addTableRow(dom.thingDetails, 'policyId', false, true, thingJson.policyId); - Utils.addTableRow(dom.thingDetails, 'definition', false, true, thingJson.definition ?? ''); - Utils.addTableRow(dom.thingDetails, 'revision', false, true, thingJson._revision); - Utils.addTableRow(dom.thingDetails, 'created', false, true, thingJson._created); - Utils.addTableRow(dom.thingDetails, 'modified', false, true, thingJson._modified); + Utils.addTableRow(dom.tbodyThingDetails, 'thingId', false, true, thingJson.thingId); + Utils.addTableRow(dom.tbodyThingDetails, 'policyId', false, true, thingJson.policyId); + Utils.addTableRow(dom.tbodyThingDetails, 'definition', false, true, thingJson.definition ?? ''); + Utils.addTableRow(dom.tbodyThingDetails, 'revision', false, true, thingJson._revision); + Utils.addTableRow(dom.tbodyThingDetails, 'created', false, true, thingJson._created); + Utils.addTableRow(dom.tbodyThingDetails, 'modified', false, true, thingJson._modified); } } function updateThingJsonEditor() { if (thingJson) { - dom.thingId.value = thingJson.thingId; - dom.buttonDeleteThing.disabled = false; + dom.crudThings.idValue = thingJson.thingId; dom.inputThingDefinition.value = thingJson.definition ?? ''; const thingCopy = JSON.parse(JSON.stringify(thingJson)); delete thingCopy['_revision']; @@ -165,37 +155,44 @@ function onThingChanged(thingJson) { delete thingCopy['_modified']; thingJsonEditor.setValue(JSON.stringify(thingCopy, null, 2), -1); } else { - dom.thingId.value = null; - dom.buttonDeleteThing.disabled = true; + dom.crudThings.idValue = null; dom.inputThingDefinition.value = null; thingJsonEditor.setValue(''); } } } -function toggleEdit() { - isEditing = !isEditing; - dom.modalThingsEdit.classList.toggle('editBackground'); - dom.divThingsCRUD.classList.toggle('editForground'); - dom.iThingsEdit.classList.toggle('bi-pencil-square'); - dom.iThingsEdit.classList.toggle('bi-x-square'); - dom.buttonThingDefinitions.disabled = !dom.buttonThingDefinitions.disabled; - dom.inputThingDefinition.disabled = !dom.inputThingDefinition.disabled; - thingJsonEditor.setReadOnly(!isEditing); - thingJsonEditor.renderer.setShowGutter(isEditing); - if (dom.thingId.value) { - dom.buttonSaveThing.disabled = !dom.buttonSaveThing.disabled; +function onEditToggle(event) { + if (event.detail) { + API.callDittoREST('GET', `/things/${Things.theThing.thingId}`, null, null, true) + .then((response) => { + eTag = response.headers.get('ETag'); + return response.json(); + }) + .then((thingJson) => { + enableDisableEditors(); + updateEditorsBeforeEdit(thingJson); + }); } else { - dom.buttonCreateThing.disabled = !dom.buttonCreateThing.disabled; - dom.thingId.disabled = !dom.thingId.disabled; - } - if (!isEditing) { + enableDisableEditors(); clearEditorsAfterCancel(); } + function enableDisableEditors() { + dom.buttonThingDefinitions.disabled = !event.detail; + dom.inputThingDefinition.disabled = !event.detail; + } + + function updateEditorsBeforeEdit(thingJson) { + dom.inputThingDefinition.value = thingJson.definition ?? ''; + thingJsonEditor.setReadOnly(!event.detail); + thingJsonEditor.renderer.setShowGutter(event.detail); + thingJsonEditor.setValue(JSON.stringify(thingJson, null, 2), -1); + } + function clearEditorsAfterCancel() { - if (dom.thingId.value && dom.thingId.value !== '') { - Things.refreshThing(dom.thingId.value, null); + if (dom.crudThings.idValue && dom.crudThings.idValue !== '') { + Things.refreshThing(dom.crudThings.idValue, null); } else { dom.inputThingDefinition.value = null; thingJsonEditor.setValue(''); diff --git a/ui/modules/things/thingsSSE.js b/ui/modules/things/thingsSSE.js index f87eb7a7039..817bc9f0c94 100644 --- a/ui/modules/things/thingsSSE.js +++ b/ui/modules/things/thingsSSE.js @@ -33,7 +33,8 @@ function onThingChanged(newThingJson, isNewThingId) { } else if (isNewThingId) { thingEventSource && thingEventSource.close(); console.log('Start SSE: ' + newThingJson.thingId); - thingEventSource = API.getEventSource(newThingJson.thingId, '&extraFields=_revision,_modified'); + thingEventSource = API.getEventSource(newThingJson.thingId, + '&fields=thingId,attributes,features,_revision,_modified'); thingEventSource.onmessage = onMessage; } } diff --git a/ui/modules/things/thingsSearch.js b/ui/modules/things/thingsSearch.js index 1eb7674bf6f..3aa3c7d87c7 100644 --- a/ui/modules/things/thingsSearch.js +++ b/ui/modules/things/thingsSearch.js @@ -117,7 +117,7 @@ export function getThings(thingIds) { * @param {String} filter Ditto search filter (rql) * @param {boolean} isMore (optional) use cursor from previous search for additional pages */ -export function searchThings(filter, isMore = false) { +function searchThings(filter, isMore = false) { document.body.style.cursor = 'progress'; const namespaces = Environments.current().searchNamespaces; diff --git a/ui/modules/utils.js b/ui/modules/utils.js index 9f5d7808540..2cbb98424ab 100644 --- a/ui/modules/utils.js +++ b/ui/modules/utils.js @@ -28,7 +28,7 @@ export function ready() { /** * Adds a table to a table element - * @param {HTMLElement} table tbody element the row is added to + * @param {HTMLTableElement} table tbody element the row is added to * @param {String} key first column text of the row. Acts as id of the row * @param {boolean} selected if true, the new row will be marked as selected * @param {boolean} withClipBoardCopy add a clipboard button at the last column of the row @@ -69,13 +69,13 @@ export function addCheckboxToRow(row, id, checked, onToggle) { /** * Adds a cell to the row including a tooltip - * @param {HTMRTableRowElement} row target row + * @param {HTMLTableRowElement} row target row * @param {String} cellContent content of new cell * @param {String} cellTooltip tooltip for new cell - * @param {integer} position optional, default -1 (add to the end) + * @param {Number} position optional, default -1 (add to the end) */ -export function addCellToRow(row, cellContent, cellTooltip, position) { - const cell = row.insertCell(position ?? -1); +export function addCellToRow(row, cellContent, cellTooltip = null, position = -1) { + const cell = row.insertCell(position); cell.innerHTML = cellContent; cell.setAttribute('data-bs-toggle', 'tooltip'); cell.title = cellTooltip ?? cellContent; @@ -149,7 +149,7 @@ export function setOptions(target, options) { * @param {array} items array of items for the drop down * @param {boolean} isHeader (optional) true to add a header line */ -export function addDropDownEntries(target, items, isHeader) { +export function addDropDownEntries(target, items, isHeader = false) { items.forEach((value) => { const li = document.createElement('li'); li.innerHTML = isHeader ? @@ -191,10 +191,11 @@ export function addTab(tabItemsNode, tabContentsNode, title, contentHTML, toolTi /** * Get the HTMLElements of all the given ids. The HTMLElements will be returned in the original object * @param {Object} domObjects object with empty keys that are used as ids of the dom elements + * @param {Element} searchRoot optional root to search in (optional for shadow dom) */ -export function getAllElementsById(domObjects) { +export function getAllElementsById(domObjects, searchRoot = document) { Object.keys(domObjects).forEach((id) => { - domObjects[id] = document.getElementById(id); + domObjects[id] = searchRoot.getElementById(id); if (!domObjects[id]) { throw new Error(`Element ${id} not found.`); } @@ -207,13 +208,13 @@ export function getAllElementsById(domObjects) { * @param {String} header Header for toast * @param {String} status Status text for toas */ -export function showError(message, header, status) { +export function showError(message, header, status = '') { const domToast = document.createElement('div'); domToast.classList.add('toast'); domToast.innerHTML = `
${header ?? 'Error'} - ${status ?? ''} + ${status}
${message}
`; @@ -319,7 +320,7 @@ export function addValidatorToTable(tableElement, inputElement) { /** * Adjust selection of a table - * @param {HTMLElement} tbody table with the data + * @param {HTMLTableElement} tbody table with the data * @param {function} condition evaluate if table row should be selected or not */ export function tableAdjustSelection(tbody, condition) { diff --git a/ui/modules/utils/crudToolbar.js b/ui/modules/utils/crudToolbar.js new file mode 100644 index 00000000000..64cd580c658 --- /dev/null +++ b/ui/modules/utils/crudToolbar.js @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +/* eslint-disable require-jsdoc */ +import * as Utils from '../utils.js'; + +class CrudToolbar extends HTMLElement { + isEditing = false; + dom = { + label: null, + inputIdValue: null, + buttonCrudEdit: null, + buttonCreate: null, + buttonUpdate: null, + buttonDelete: null, + iEdit: null, + divRoot: null, + }; + + get idValue() { + return this.dom.inputIdValue.value; + } + + set idValue(newValue) { + this.dom.inputIdValue.value = newValue; + this.dom.buttonDelete.disabled = (newValue === null); + } + + get editDisabled() { + return this.dom.buttonCrudEdit.disabled; + } + + set editDisabled(newValue) { + this.dom.buttonCrudEdit.disabled = newValue; + } + + get validationElement() { + return this.dom.inputIdValue; + } + + constructor() { + super(); + this.attachShadow({mode: 'open'}); + } + + connectedCallback() { + this.shadowRoot.append(document.getElementById('templateCrudToolbar').content.cloneNode(true)); + setTimeout(() => { + Utils.getAllElementsById(this.dom, this.shadowRoot); + this.dom.buttonCrudEdit.onclick = () => this.toggleEdit(); + this.dom.label.innerText = this.getAttribute('label') || 'Label'; + this.dom.buttonCreate.onclick = this.eventDispatcher('onCreateClick'); + this.dom.buttonUpdate.onclick = this.eventDispatcher('onUpdateClick'); + this.dom.buttonDelete.onclick = this.eventDispatcher('onDeleteClick'); + }); + }; + + eventDispatcher(eventName) { + return () => { + this.dispatchEvent(new CustomEvent(eventName, { + composed: true, + })); + }; + } + + toggleEdit() { + this.isEditing = !this.isEditing; + document.getElementById('modalCrudEdit').classList.toggle('editBackground'); + this.dom.divRoot.classList.toggle('editForground'); + this.dom.iEdit.classList.toggle('bi-pencil-square'); + this.dom.iEdit.classList.toggle('bi-x-square'); + this.dom.buttonCrudEdit.title = this.isEditing ? 'Cancel' : 'Edit'; + this.dom.buttonCreate.disabled = !(this.isEditing && !this.dom.inputIdValue.value); + this.dom.buttonUpdate.disabled = !(this.isEditing && this.dom.inputIdValue.value); + if (this.isEditing || !this.dom.inputIdValue.value) { + this.dom.buttonDelete.disabled = true; + } + this.dom.inputIdValue.disabled = !(this.isEditing && !this.dom.inputIdValue.value); + this.dispatchEvent(new CustomEvent('onEditToggle', { + composed: true, + detail: this.isEditing, + })); + } +} + +customElements.define('crud-toolbar', CrudToolbar); From 4d93fb77de35aad1ecdf43b37dc8e50e263d5e68 Mon Sep 17 00:00:00 2001 From: thfries Date: Fri, 6 Jan 2023 19:05:28 +0100 Subject: [PATCH 007/173] Explorer UI - Apply new edit style to environments Signed-off-by: thfries --- ui/modules/environments/environments.html | 63 +++------ ui/modules/environments/environments.js | 162 +++++++++++++--------- ui/modules/utils/crudToolbar.js | 3 +- 3 files changed, 122 insertions(+), 106 deletions(-) diff --git a/ui/modules/environments/environments.html b/ui/modules/environments/environments.html index 51d7858ec17..907151ff786 100644 --- a/ui/modules/environments/environments.html +++ b/ui/modules/environments/environments.html @@ -35,55 +35,34 @@
Environments
-
- - - - - -
-
-
- - -
-
- - -
-
- - -
+ +
+ + +
+
+ + +
+
+ + +
+
-
-
-
- -
+
-
+
diff --git a/ui/modules/environments/environments.js b/ui/modules/environments/environments.js index e9c7faf15dc..ee51906b0d3 100644 --- a/ui/modules/environments/environments.js +++ b/ui/modules/environments/environments.js @@ -29,16 +29,13 @@ let settingsEditor; let dom = { environmentSelector: null, - buttonCreateEnvironment: null, - buttonDeleteEnvironment: null, - buttonUpdateFields: null, - buttonUpdateJson: null, - inputEnvironmentName: null, + tbodyEnvironments: null, + tableValidationEnvironments: null, + crudEnvironmentFields: null, + crudEnvironmentJson: null, inputApiUri: null, inputSearchNamespaces: null, selectDittoVersion: null, - tbodyEnvironments: null, - tableValidationEnvironments: null, }; let observers = []; @@ -91,73 +88,101 @@ export async function ready() { urlSearchParams = new URLSearchParams(window.location.search); environments = await loadEnvironmentTemplates(); - settingsEditor = ace.edit('settingsEditor'); - settingsEditor.session.setMode('ace/mode/json'); - - dom.buttonUpdateJson.onclick = () => { - Utils.assert(selectedEnvName, 'No environment selected', dom.tableValidationEnvironments); - environments[selectedEnvName] = JSON.parse(settingsEditor.getValue()); - environmentsJsonChanged(); - }; + settingsEditor = Utils.createAceEditor('settingsEditor', 'ace/mode/json', true); - dom.buttonUpdateFields.onclick = () => { - Utils.assert(selectedEnvName, 'No environment selected', dom.tableValidationEnvironments); - if (selectedEnvName !== dom.inputEnvironmentName.value) { - environments[dom.inputEnvironmentName.value] = environments[selectedEnvName]; - delete environments[selectedEnvName]; - selectedEnvName = dom.inputEnvironmentName.value; - } - environments[selectedEnvName].api_uri = dom.inputApiUri.value; - environments[selectedEnvName].searchNamespaces = dom.inputSearchNamespaces.value; - environments[selectedEnvName].ditto_version = dom.selectDittoVersion.value; - environmentsJsonChanged(); - }; + dom.tbodyEnvironments.addEventListener('click', onEnvironmentsTableClick); - dom.tbodyEnvironments.addEventListener('click', (event) => { - if (event.target && event.target.tagName === 'TD') { - if (selectedEnvName && selectedEnvName === event.target.parentNode.id) { - selectedEnvName = null; - } else { - selectedEnvName = event.target.parentNode.id; - } - updateEnvEditors(); - } - }); + dom.crudEnvironmentJson.addEventListener('onCreateClick', onCreateEnvironmentClick); + dom.crudEnvironmentJson.addEventListener('onUpdateClick', onUpdateEnvironmentClick); + dom.crudEnvironmentJson.addEventListener('onDeleteClick', onDeleteEnvironmentClick); + dom.crudEnvironmentJson.addEventListener('onEditToggle', onEditToggle); - dom.buttonCreateEnvironment.onclick = () => { - Utils.assert(dom.inputEnvironmentName.value, 'Please provide an environment name', dom.inputEnvironmentName); - Utils.assert(!environments[dom.inputEnvironmentName.value], 'Name already used', dom.inputEnvironmentName); - environments[dom.inputEnvironmentName.value] = new Environment({ - api_uri: dom.inputApiUri.value ? dom.inputApiUri.value : '', - ditto_version: dom.selectDittoVersion.value ? dom.selectDittoVersion.value : '3', - }); - selectedEnvName = dom.inputEnvironmentName.value; - environmentsJsonChanged(); - }; + dom.crudEnvironmentFields.addEventListener('onCreateClick', onCreateEnvironmentClick); + dom.crudEnvironmentFields.addEventListener('onUpdateClick', onUpdateEnvironmentClick); + dom.crudEnvironmentFields.addEventListener('onDeleteClick', onDeleteEnvironmentClick); + dom.crudEnvironmentFields.addEventListener('onEditToggle', onEditToggle); - dom.buttonDeleteEnvironment.onclick = () => { - Utils.assert(selectedEnvName, 'No environment selected', dom.tableValidationEnvironments); - Utils.assert(Object.keys(environments).length >= 2, 'At least one environment is required', - dom.tableValidationEnvironments); - delete environments[selectedEnvName]; - selectedEnvName = null; - environmentsJsonChanged(); - }; + dom.environmentSelector.onchange = onEnvironmentSelectorChange; // Ensure that ace editor to refresh when updated while hidden document.querySelector('a[data-bs-target="#tabEnvJson"]').addEventListener('shown.bs.tab', () => { settingsEditor.renderer.updateFull(); }); - dom.environmentSelector.onchange = () => { - urlSearchParams.set(URL_PRIMARY_ENVIRONMENT_NAME, dom.environmentSelector.value); - window.history.replaceState({}, '', `${window.location.pathname}?${urlSearchParams}`); - notifyAll(); - }; environmentsJsonChanged(); } +function onEnvironmentSelectorChange() { + urlSearchParams.set(URL_PRIMARY_ENVIRONMENT_NAME, dom.environmentSelector.value); + window.history.replaceState({}, '', `${window.location.pathname}?${urlSearchParams}`); + notifyAll(); +} + +function onDeleteEnvironmentClick() { + Utils.assert(selectedEnvName, 'No environment selected', dom.tableValidationEnvironments); + Utils.assert(Object.keys(environments).length >= 2, 'At least one environment is required', + dom.tableValidationEnvironments); + delete environments[selectedEnvName]; + selectedEnvName = null; + environmentsJsonChanged(); +} + +function onCreateEnvironmentClick(event) { + Utils.assert(event.target.idValue, + 'Environment name must not be empty', + event.target.validationElement); + Utils.assert(!environments[event.target.idValue], + 'Environment name already used', + event.target.validationElement); + + if (event.target === dom.crudEnvironmentFields) { + environments[event.target.idValue] = new Environment({ + api_uri: dom.inputApiUri.value ?? '', + searchNamespaces: dom.inputSearchNamespaces.value ?? '', + ditto_version: dom.selectDittoVersion.value ? dom.selectDittoVersion.value : '3', + }); + } else { + environments[event.target.idValue] = new Environment(JSON.parse(settingsEditor.getValue())); + } + + selectedEnvName = event.target.idValue; + event.target.toggleEdit(); + environmentsJsonChanged(); +} + +function onEnvironmentsTableClick(event) { + if (event.target && event.target.tagName === 'TD') { + if (selectedEnvName && selectedEnvName === event.target.parentNode.id) { + selectedEnvName = null; + } else { + selectedEnvName = event.target.parentNode.id; + } + updateEnvEditors(); + } +} + +function onUpdateEnvironmentClick(event) { + if (selectedEnvName !== event.target.idValue) { + changeEnvironmentName(); + } + if (event.target === dom.crudEnvironmentFields) { + environments[selectedEnvName].api_uri = dom.inputApiUri.value; + environments[selectedEnvName].searchNamespaces = dom.inputSearchNamespaces.value; + environments[selectedEnvName].ditto_version = dom.selectDittoVersion.value; + } else { + environments[selectedEnvName] = JSON.parse(settingsEditor.getValue()); + } + event.target.toggleEdit(); + environmentsJsonChanged(); + + function changeEnvironmentName() { + environments[event.target.idValue] = environments[selectedEnvName]; + delete environments[selectedEnvName]; + selectedEnvName = event.target.idValue; + } +} + export function environmentsJsonChanged(modifiedField) { localStorage.setItem(STORAGE_KEY, JSON.stringify(environments)); @@ -192,13 +217,15 @@ export function environmentsJsonChanged(modifiedField) { function updateEnvEditors() { if (selectedEnvName) { const selectedEnvironment = environments[selectedEnvName]; - dom.inputEnvironmentName.value = selectedEnvName; + dom.crudEnvironmentFields.idValue = selectedEnvName; + dom.crudEnvironmentJson.idValue = selectedEnvName; settingsEditor.setValue(JSON.stringify(selectedEnvironment, null, 2), -1); dom.inputApiUri.value = selectedEnvironment.api_uri; dom.inputSearchNamespaces.value = selectedEnvironment.searchNamespaces ?? ''; dom.selectDittoVersion.value = selectedEnvironment.ditto_version ? selectedEnvironment.ditto_version : '3'; } else { - dom.inputEnvironmentName.value = null; + dom.crudEnvironmentFields.idValue = null; + dom.crudEnvironmentJson.idValue = null; settingsEditor.setValue(''); dom.inputApiUri.value = null; dom.inputSearchNamespaces.value = null; @@ -276,4 +303,13 @@ async function loadEnvironmentTemplates() { } } - +function onEditToggle(event) { + dom.inputApiUri.disabled = !event.detail; + dom.inputSearchNamespaces.disabled = !event.detail; + dom.selectDittoVersion.disabled = !event.detail; + settingsEditor.setReadOnly(!event.detail); + settingsEditor.renderer.setShowGutter(event.detail); + if (!event.detail) { + updateEnvEditors(); + } +} diff --git a/ui/modules/utils/crudToolbar.js b/ui/modules/utils/crudToolbar.js index 64cd580c658..48cfda086cd 100644 --- a/ui/modules/utils/crudToolbar.js +++ b/ui/modules/utils/crudToolbar.js @@ -84,7 +84,8 @@ class CrudToolbar extends HTMLElement { if (this.isEditing || !this.dom.inputIdValue.value) { this.dom.buttonDelete.disabled = true; } - this.dom.inputIdValue.disabled = !(this.isEditing && !this.dom.inputIdValue.value); + const allowIdChange = this.isEditing && (!this.dom.inputIdValue.value || this.hasAttribute('allowIdChange')); + this.dom.inputIdValue.disabled = !allowIdChange; this.dispatchEvent(new CustomEvent('onEditToggle', { composed: true, detail: this.isEditing, From ad71d59b6f1d841e05a57de70b9f52dfe5acdecd Mon Sep 17 00:00:00 2001 From: thfries Date: Sun, 8 Jan 2023 09:35:17 +0100 Subject: [PATCH 008/173] Explorer UI - Add new edit style to connections Signed-off-by: thfries --- ui/main.js | 4 + ui/modules/api.js | 80 +++--- ui/modules/connections/connections.html | 77 +++--- ui/modules/connections/connections.js | 270 +++---------------- ui/modules/connections/connectionsCRUD.js | 204 ++++++++++++++ ui/modules/connections/connectionsMonitor.js | 145 ++++++++++ ui/modules/things/attributes.js | 3 +- ui/modules/things/features.js | 13 +- ui/modules/things/thingsCRUD.js | 11 +- 9 files changed, 472 insertions(+), 335 deletions(-) create mode 100644 ui/modules/connections/connectionsCRUD.js create mode 100644 ui/modules/connections/connectionsMonitor.js diff --git a/ui/main.js b/ui/main.js index cddd2a77f83..598b9983bd3 100644 --- a/ui/main.js +++ b/ui/main.js @@ -24,6 +24,8 @@ import * as ThingsSearch from './modules/things/thingsSearch.js'; import * as ThingsCRUD from './modules/things/thingsCRUD.js'; import * as ThingsSSE from './modules/things/thingsSSE.js'; import * as Connections from './modules/connections/connections.js'; +import * as ConnectionsCRUD from './modules/connections/connectionsCRUD.js'; +import * as ConnectionsMonitor from './modules/connections/connectionsMonitor.js'; import * as Policies from './modules/policies/policies.js'; import * as API from './modules/api.js'; import * as Utils from './modules/utils.js'; @@ -57,6 +59,8 @@ document.addEventListener('DOMContentLoaded', async function() { await FeatureMessages.ready(); Policies.ready(); Connections.ready(); + ConnectionsCRUD.ready(); + ConnectionsMonitor.ready(); Authorization.ready(); await Environments.ready(); diff --git a/ui/modules/api.js b/ui/modules/api.js index bc639d0f08f..95b3853aa9f 100644 --- a/ui/modules/api.js +++ b/ui/modules/api.js @@ -360,11 +360,9 @@ export function getEventSource(thingId, urlParams) { export async function callConnectionsAPI(operation, successCallback, connectionId, connectionJson, command) { Utils.assert((env() !== 'things' || Environments.current().solutionId), 'No solutionId configured in environment'); const params = config[env()][operation]; - // if (param && param.charAt(0) === '"' && param.charAt(str.length -1) === '"') { - // param = param.substr(1, param.length -2); - // }; + let response; try { - const response = await fetch(Environments.current().api_uri + params.path.replace('{{solutionId}}', + response = await fetch(Environments.current().api_uri + params.path.replace('{{solutionId}}', Environments.current().solutionId).replace('{{connectionId}}', connectionId), { method: params.method, headers: { @@ -378,46 +376,48 @@ export async function callConnectionsAPI(operation, successCallback, connectionI .replace('{{command}}', command) : connectionJson ? JSON.stringify(connectionJson) : command, }); - if (!response.ok) { - response.json() - .then((dittoErr) => { - Utils.showError(dittoErr.description, dittoErr.message, dittoErr.status); - }) - .catch((err) => { - Utils.showError('No error details from Ditto', response.statusText, response.status); - }); - throw new Error('An error occured: ' + response.status); - } - if (operation !== 'connectionCommand' && response.status !== 204) { - document.body.style.cursor = 'progress'; - response.json().then((data) => { - if (data && data['?'] && data['?']['?'].status >= 400) { - const dittoErr = data['?']['?'].payload; - Utils.showError(dittoErr.description, dittoErr.message, dittoErr.status); - } else { - if (params.unwrapJsonPath) { - params.unwrapJsonPath.split('.').forEach(function(node) { - if (node === '?') { - node = Object.keys(data)[0]; - } - data = data[node]; - }); - } - successCallback(data); - } - }).catch((error) => { - Utils.showError('Error calling connections API', error); - throw error; - }).finally(() => { - document.body.style.cursor = 'default'; - }); - } else { - successCallback && successCallback(); - } } catch (err) { Utils.showError(err); throw err; } + + if (!response.ok) { + response.json() + .then((dittoErr) => { + Utils.showError(dittoErr.description, dittoErr.message, dittoErr.status); + }) + .catch((err) => { + Utils.showError('No error details from Ditto', response.statusText, response.status); + }); + throw new Error('An error occured: ' + response.status); + } + if (operation !== 'connectionCommand' && response.status !== 204) { + document.body.style.cursor = 'progress'; + response.json() + .then((data) => { + if (data && data['?'] && data['?']['?'].status >= 400) { + const dittoErr = data['?']['?'].payload; + Utils.showError(dittoErr.description, dittoErr.message, dittoErr.status); + } else { + if (params.unwrapJsonPath) { + params.unwrapJsonPath.split('.').forEach(function(node) { + if (node === '?') { + node = Object.keys(data)[0]; + } + data = data[node]; + }); + } + successCallback(data); + } + }).catch((error) => { + Utils.showError('Error calling connections API', error); + throw error; + }).finally(() => { + document.body.style.cursor = 'default'; + }); + } else { + successCallback && successCallback(); + } } export function env() { diff --git a/ui/modules/connections/connections.html b/ui/modules/connections/connections.html index c7baa890f6f..a191afa1ff4 100644 --- a/ui/modules/connections/connections.html +++ b/ui/modules/connections/connections.html @@ -28,7 +28,7 @@
-
+
@@ -41,49 +41,40 @@
-
-
Connection
-
- - - - - -
-
- -
- - +
+
CRUD Connection
+ +
+ +
+ + +
+
- -
-
- -
-
-
-
-
-
-
-
Incoming JavaScript Mapping
-
-
-
-
Outgoing JavaScript Mapping
-
-
-
+
+ +
+
+
+
+
+
+
+
+
+
Incoming JavaScript Mapping
+
+
+
+
Outgoing JavaScript Mapping
+
+
+
+
+
+
diff --git a/ui/modules/connections/connections.js b/ui/modules/connections/connections.js index d84810e3e0a..4e00cb4bd23 100644 --- a/ui/modules/connections/connections.js +++ b/ui/modules/connections/connections.js @@ -10,251 +10,52 @@ * * SPDX-License-Identifier: EPL-2.0 */ - -import * as API from '../api.js'; -import * as Environments from '../environments/environments.js'; /* eslint-disable prefer-const */ /* eslint-disable max-len */ /* eslint-disable no-invalid-this */ /* eslint-disable require-jsdoc */ +import * as API from '../api.js'; +import * as Environments from '../environments/environments.js'; import * as Utils from '../utils.js'; +const observers = []; + +export function addChangeListener(observer) { + observers.push(observer); +} + +function notifyAll(connection, isNewConnection) { + observers.forEach((observer) => observer.call(null, connection, isNewConnection)); +} + let dom = { - ulConnectionTemplates: null, - inputConnectionTemplate: null, - inputConnectionId: null, tbodyConnections: null, - tbodyConnectionLogs: null, - tbodyConnectionMetrics: null, buttonLoadConnections: null, - buttonCreateConnection: null, - buttonSaveConnection: null, - buttonDeleteConnection: null, - buttonRetrieveConnectionStatus: null, - buttonRetrieveConnectionLogs: null, - buttonEnableConnectionLogs: null, - buttonResetConnectionLogs: null, - buttonRetrieveConnectionMetrics: null, - buttonResetConnectionMetrics: null, tabConnections: null, collapseConnections: null, - editorValidationConnection: null, tableValidationConnections: null, }; -let connectionEditor; -let incomingEditor; -let outgoingEditor; -let connectionLogDetail; -let connectionStatusDetail; - -let theConnection; let selectedConnectionId; -let connectionLogs; - -let connectionTemplates; export function ready() { Environments.addChangeListener(onEnvironmentChanged); Utils.getAllElementsById(dom); - connectionEditor = Utils.createAceEditor('connectionEditor', 'ace/mode/json'); - incomingEditor = Utils.createAceEditor('connectionIncomingScript', 'ace/mode/javascript'); - outgoingEditor = Utils.createAceEditor('connectionOutgoingScript', 'ace/mode/javascript'); - connectionLogDetail = Utils.createAceEditor('connectionLogDetail', 'ace/mode/json', true); - connectionStatusDetail = Utils.createAceEditor('connectionStatusDetail', 'ace/mode/json', true); - Utils.addValidatorToTable(dom.tbodyConnections, dom.tableValidationConnections); - loadConnectionTemplates(); - - dom.buttonLoadConnections.onclick = loadConnections; dom.tabConnections.onclick = onTabActivated; - - dom.tbodyConnections.addEventListener('click', (event) => { - if (event.target && event.target.tagName === 'TD') { - if (selectedConnectionId === event.target.parentNode.id) { - selectedConnectionId = null; - setConnection(null); - } else { - selectedConnectionId = event.target.parentNode.id; - API.callConnectionsAPI('retrieveConnection', setConnection, selectedConnectionId); - } - } - }); - - dom.ulConnectionTemplates.addEventListener('click', (event) => { - dom.inputConnectionTemplate.value = event.target.textContent; - const templateConnection = {}; - if (API.env() !== 'things') { - templateConnection.id = Math.random().toString(36).replace('0.', ''); - } - const newConnection = JSON.parse(JSON.stringify( - connectionTemplates[dom.inputConnectionTemplate.value])); - - const mergedConnection = {...templateConnection, ...newConnection}; - setConnection(mergedConnection, true); - dom.editorValidationConnection.classList.remove('is-invalid'); - connectionEditor.session.getUndoManager().markClean(); - }); - - incomingEditor.on('blur', function() { - initializeMappings(theConnection); - theConnection.mappingDefinitions.javascript.options.incomingScript = incomingEditor.getValue(); - connectionEditor.setValue(JSON.stringify(theConnection, null, 2), -1); - }); - outgoingEditor.on('blur', function() { - initializeMappings(theConnection); - theConnection.mappingDefinitions.javascript.options.outgoingScript = outgoingEditor.getValue(); - connectionEditor.setValue(JSON.stringify(theConnection, null, 2), -1); - }); - connectionEditor.on('blur', function() { - theConnection = JSON.parse(connectionEditor.getValue()); - }); - - connectionEditor.on('input', () => { - if (!connectionEditor.session.getUndoManager().isClean()) { - dom.inputConnectionTemplate.value = null; - dom.editorValidationConnection.classList.remove('is-invalid'); - } - }); - - dom.buttonCreateConnection.onclick = () => { - Utils.assert(theConnection, 'Please enter a connection configuration (select a template as a basis)', dom.editorValidationConnection); - if (API.env() === 'ditto_2') { - selectedConnectionId = theConnection.id; - } else { - delete theConnection.id; - } - API.callConnectionsAPI('createConnection', loadConnections, null, theConnection); - }; - - dom.buttonSaveConnection.onclick = () => { - Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); - API.callConnectionsAPI('modifyConnection', loadConnections, selectedConnectionId, theConnection); - }; - - dom.buttonDeleteConnection.onclick = () => { - Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); - Utils.confirm(`Are you sure you want to delete connection
'${theConnection.name}'?`, 'Delete', () => { - API.callConnectionsAPI('deleteConnection', () => { - setConnection(null); - loadConnections(); - }, - selectedConnectionId); - }); - }; - - // Status -------------- - - dom.buttonRetrieveConnectionStatus.onclick = retrieveConnectionStatus; - document.querySelector('a[data-bs-target="#tabConnectionStatus"]').onclick = retrieveConnectionStatus; - - // Logs -------------- - - dom.buttonEnableConnectionLogs.onclick = () => { - Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); - API.callConnectionsAPI('connectionCommand', retrieveConnectionLogs, dom.inputConnectionId.value, null, 'connectivity.commands:enableConnectionLogs'); - }; - - dom.buttonResetConnectionLogs.onclick = () => { - Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); - API.callConnectionsAPI('connectionCommand', retrieveConnectionLogs, dom.inputConnectionId.value, null, 'connectivity.commands:resetConnectionLogs'); - }; - - dom.buttonRetrieveConnectionLogs.onclick = retrieveConnectionLogs; - - dom.tbodyConnectionLogs.addEventListener('click', (event) => { - connectionLogDetail.setValue(JSON.stringify(connectionLogs[event.target.parentNode.rowIndex - 1], null, 2), -1); - }); - - // Metrics --------------- - - dom.buttonRetrieveConnectionMetrics.onclick = retrieveConnectionMetrics; - document.querySelector('a[data-bs-target="#tabConnectionMetrics"]').onclick = retrieveConnectionMetrics; - - dom.buttonResetConnectionMetrics.onclick = () => { - Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); - API.callConnectionsAPI('connectionCommand', null, dom.inputConnectionId.value, null, 'connectivity.commands:resetConnectionMetrics'); - }; -} - -function retrieveConnectionMetrics() { - Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); - dom.tbodyConnectionMetrics.innerHTML = ''; - API.callConnectionsAPI('retrieveConnectionMetrics', (response) => { - if (response.connectionMetrics) { - Object.keys(response.connectionMetrics).forEach((direction) => { - Object.keys(response.connectionMetrics[direction]).forEach((type) => { - let entry = response.connectionMetrics[direction][type]; - Utils.addTableRow(dom.tbodyConnectionMetrics, direction, false, false, type, 'success', entry.success.PT1M, entry.success.PT1H, entry.success.PT24H); - Utils.addTableRow(dom.tbodyConnectionMetrics, direction, false, false, type, 'failure', entry.failure.PT1M, entry.failure.PT1H, entry.failure.PT24H); - }); - }); - } - }, - dom.inputConnectionId.value); -} - -function retrieveConnectionStatus() { - Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); - API.callConnectionsAPI('retrieveStatus', (connectionStatus) => { - connectionStatusDetail.setValue(JSON.stringify(connectionStatus, null, 2), -1); - }, - dom.inputConnectionId.value); -} - -function retrieveConnectionLogs() { - Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); - dom.tbodyConnectionLogs.innerHTML = ''; - connectionLogDetail.setValue(''); - API.callConnectionsAPI('retrieveConnectionLogs', (response) => { - connectionLogs = response.connectionLogs; - adjustEnableButton(response); - response.connectionLogs.forEach((entry) => { - Utils.addTableRow(dom.tbodyConnectionLogs, Utils.formatDate(entry.timestamp, true), false, false, entry.type, entry.level); - }); - dom.tbodyConnectionLogs.scrollTop = dom.tbodyConnectionLogs.scrollHeight - dom.tbodyConnectionLogs.clientHeight; - }, - dom.inputConnectionId.value); - - function adjustEnableButton(response) { - if (response.enabledUntil) { - dom.buttonEnableConnectionLogs.querySelector('i').classList.replace('bi-toggle-off', 'bi-toggle-on'); - dom.buttonEnableConnectionLogs.setAttribute('title', `Enabled until ${Utils.formatDate(response.enabledUntil)}`); - } else { - dom.buttonEnableConnectionLogs.querySelector('i').classList.replace('bi-toggle-on', 'bi-toggle-off'); - dom.buttonEnableConnectionLogs.setAttribute('title', 'Click to enable connection logs for the selected connection'); - } - } + dom.buttonLoadConnections.onclick = loadConnections; + dom.tbodyConnections.addEventListener('click', onConnectionsTableClick); } -function setConnection(connection, isNewConnection) { - theConnection = connection; - incomingEditor.setValue(''); - outgoingEditor.setValue(''); - if (theConnection) { - dom.inputConnectionId.value = theConnection.id ? theConnection.id : null; - connectionEditor.setValue(JSON.stringify(theConnection, null, 2), -1); - if (theConnection.mappingDefinitions && theConnection.mappingDefinitions.javascript) { - incomingEditor.setValue(theConnection.mappingDefinitions.javascript.options.incomingScript, -1); - outgoingEditor.setValue(theConnection.mappingDefinitions.javascript.options.outgoingScript, -1); - } - } else { - dom.inputConnectionId.value = null; - connectionEditor.setValue(''); - } - connectionStatusDetail.setValue(''); - connectionLogDetail.setValue(''); - dom.tbodyConnectionMetrics.innerHTML = ''; - dom.tbodyConnectionLogs.innerHTML = ''; - if (!isNewConnection && theConnection && theConnection.id) { - retrieveConnectionLogs(); - } +export function setConnection(connection, isNewConnection) { + selectedConnectionId = connection ? connection.id : null; + notifyAll(connection, isNewConnection); } -function loadConnections() { +export function loadConnections() { dom.tbodyConnections.innerHTML = ''; let connectionSelected = false; API.callConnectionsAPI('listConnections', (connections) => { @@ -287,14 +88,16 @@ function loadConnections() { }); } -function loadConnectionTemplates() { - fetch('templates/connectionTemplates.json') - .then((response) => { - response.json().then((loadedTemplates) => { - connectionTemplates = loadedTemplates; - Utils.addDropDownEntries(dom.ulConnectionTemplates, Object.keys(connectionTemplates)); - }); - }); +function onConnectionsTableClick(event) { + if (event.target && event.target.tagName === 'TD') { + if (selectedConnectionId === event.target.parentNode.id) { + selectedConnectionId = null; + setConnection(null); + } else { + selectedConnectionId = event.target.parentNode.id; + API.callConnectionsAPI('retrieveConnection', setConnection, selectedConnectionId); + } + } } let viewDirty = false; @@ -308,23 +111,10 @@ function onTabActivated() { function onEnvironmentChanged() { if (dom.collapseConnections.classList.contains('show')) { + selectedConnectionId = null; + setConnection(null); loadConnections(); } else { viewDirty = true; } } - -function initializeMappings(connection) { - if (!connection['mappingDefinitions']) { - connection.mappingDefinitions = { - javascript: { - mappingEngine: 'JavaScript', - options: { - incomingScript: '', - outgoingScript: '', - }, - }, - }; - } -} - diff --git a/ui/modules/connections/connectionsCRUD.js b/ui/modules/connections/connectionsCRUD.js new file mode 100644 index 00000000000..731808e90f5 --- /dev/null +++ b/ui/modules/connections/connectionsCRUD.js @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +import * as API from '../api.js'; +import * as Utils from '../utils.js'; +import * as Connections from './connections.js'; +/* eslint-disable prefer-const */ +/* eslint-disable max-len */ +/* eslint-disable no-invalid-this */ +/* eslint-disable require-jsdoc */ + +let dom = { + buttonConnectionTemplates: null, + ulConnectionTemplates: null, + inputConnectionTemplate: null, + crudConnection: null, + editorValidationConnection: null, +}; + +let connectionEditor; +let incomingEditor; +let outgoingEditor; + +let theConnection; +let hasErrors; + +let connectionTemplates; + +export function ready() { + Connections.addChangeListener(setConnection); + + Utils.getAllElementsById(dom); + + loadConnectionTemplates(); + + connectionEditor = Utils.createAceEditor('connectionEditor', 'ace/mode/json', true); + incomingEditor = Utils.createAceEditor('connectionIncomingScript', 'ace/mode/javascript', true); + outgoingEditor = Utils.createAceEditor('connectionOutgoingScript', 'ace/mode/javascript', true); + + dom.ulConnectionTemplates.addEventListener('click', onConnectionTemplatesClick); + + incomingEditor.on('blur', onScriptEditorBlur(incomingEditor, 'incomingScript')); + outgoingEditor.on('blur', onScriptEditorBlur(outgoingEditor, 'outgoingScript')); + + connectionEditor.on('input', onConnectionEditorInput); + connectionEditor.getSession().on('changeAnnotation', onConnectionEditorChangeAnnotation); + + dom.crudConnection.addEventListener('onEditToggle', onEditToggle); + dom.crudConnection.addEventListener('onCreateClick', onCreateConnectionClick); + dom.crudConnection.addEventListener('onUpdateClick', onUpdateConnectionClick); + dom.crudConnection.addEventListener('onDeleteClick', onDeleteConnectionClick); +} + +function onScriptEditorBlur(scriptEditor, fieldName) { + return () => { + if (!crudConnection.isEditing || hasErrors) { + return; + } + const editConnection = JSON.parse(connectionEditor.getValue()); + initializeMappings(editConnection); + editConnection.mappingDefinitions.javascript.options[fieldName] = scriptEditor.getValue(); + connectionEditor.setValue(JSON.stringify(editConnection, null, 2), -1); + }; + + function initializeMappings(connection) { + if (!connection['mappingDefinitions']) { + connection.mappingDefinitions = { + javascript: { + mappingEngine: 'JavaScript', + options: { + incomingScript: '', + outgoingScript: '', + }, + }, + }; + } + } +} + +function onConnectionEditorInput() { + if (!connectionEditor.session.getUndoManager().isClean()) { + dom.inputConnectionTemplate.value = null; + dom.editorValidationConnection.classList.remove('is-invalid'); + } +} + +function onConnectionEditorChangeAnnotation() { + hasErrors = connectionEditor.getSession().getAnnotations().filter((a) => a.type === 'error').length > 0; + incomingEditor.setReadOnly(hasErrors); + outgoingEditor.setReadOnly(hasErrors); +} + +function onUpdateConnectionClick() { + Utils.assert(!hasErrors, 'Errors in connection json', dom.editorValidationConnection); + theConnection = JSON.parse(connectionEditor.getValue()); + API.callConnectionsAPI( + 'modifyConnection', + () => { + Connections.loadConnections(), + dom.crudConnection.toggleEdit(); + }, + dom.crudConnection.idValue, + theConnection, + ); +} + +function onDeleteConnectionClick() { + Utils.confirm(`Are you sure you want to delete connection
'${theConnection.name}'?`, 'Delete', () => { + API.callConnectionsAPI( + 'deleteConnection', + () => { + Connections.setConnection(null); + Connections.loadConnections(); + }, + dom.crudConnection.idValue, + ); + }); +} + +function onCreateConnectionClick() { + Utils.assert(connectionEditor.getValue() !== '', 'Please enter a connection configuration (select a template as a basis)', dom.editorValidationConnection); + Utils.assert(!hasErrors, 'Errors in connection json', dom.editorValidationConnection); + const newConnection = JSON.parse(connectionEditor.getValue()); + if (API.env() === 'ditto_2') { + newConnection.id = Math.random().toString(36).replace('0.', ''); + } else { + delete newConnection.id; + }; + API.callConnectionsAPI( + 'createConnection', + (connection) => { + Connections.setConnection(connection, true); + Connections.loadConnections(); + dom.crudConnection.toggleEdit(); + }, + null, + newConnection, + ); +} + +function onConnectionTemplatesClick(event) { + dom.inputConnectionTemplate.value = event.target.textContent; + const newConnection = JSON.parse(JSON.stringify(connectionTemplates[dom.inputConnectionTemplate.value])); + setConnection(newConnection); + dom.editorValidationConnection.classList.remove('is-invalid'); + connectionEditor.session.getUndoManager().markClean(); +} + +function loadConnectionTemplates() { + fetch('templates/connectionTemplates.json') + .then((response) => { + response.json().then((loadedTemplates) => { + connectionTemplates = loadedTemplates; + Utils.addDropDownEntries(dom.ulConnectionTemplates, Object.keys(connectionTemplates)); + }); + }); +} + +function setConnection(connection) { + theConnection = connection; + incomingEditor.setValue(''); + outgoingEditor.setValue(''); + if (theConnection) { + dom.crudConnection.idValue = theConnection.id ? theConnection.id : null; + connectionEditor.setValue(JSON.stringify(theConnection, null, 2), -1); + if (theConnection.mappingDefinitions && theConnection.mappingDefinitions.javascript) { + incomingEditor.setValue(theConnection.mappingDefinitions.javascript.options.incomingScript, -1); + outgoingEditor.setValue(theConnection.mappingDefinitions.javascript.options.outgoingScript, -1); + } + } else { + dom.crudConnection.idValue = null; + connectionEditor.setValue(''); + } +} + +function onEditToggle(event) { + const isEditing = event.detail; + dom.buttonConnectionTemplates.disabled = !isEditing; + connectionEditor.setReadOnly(!isEditing); + connectionEditor.renderer.setShowGutter(isEditing); + incomingEditor.setReadOnly(!isEditing); + incomingEditor.renderer.setShowGutter(isEditing); + outgoingEditor.setReadOnly(!isEditing); + outgoingEditor.renderer.setShowGutter(isEditing); + if (!isEditing) { + if (dom.crudConnection.idValue && dom.crudConnection.idValue !== '') { + setConnection(theConnection); + } else { + setConnection(null); + } + } +} + + diff --git a/ui/modules/connections/connectionsMonitor.js b/ui/modules/connections/connectionsMonitor.js new file mode 100644 index 00000000000..862daf4b59d --- /dev/null +++ b/ui/modules/connections/connectionsMonitor.js @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +import * as API from '../api.js'; +import * as Utils from '../utils.js'; +import * as Connections from './connections.js'; +/* eslint-disable prefer-const */ +/* eslint-disable max-len */ +/* eslint-disable no-invalid-this */ +/* eslint-disable require-jsdoc */ + +let dom = { + tbodyConnectionLogs: null, + tbodyConnectionMetrics: null, + buttonRetrieveConnectionStatus: null, + buttonRetrieveConnectionLogs: null, + buttonEnableConnectionLogs: null, + buttonResetConnectionLogs: null, + buttonRetrieveConnectionMetrics: null, + buttonResetConnectionMetrics: null, + tableValidationConnections: null, +}; + +let connectionLogs; +let connectionLogDetail; + +let connectionStatusDetail; + +let selectedConnectionId; + +export function ready() { + Connections.addChangeListener(onConnectionChange); + + Utils.getAllElementsById(dom); + + connectionLogDetail = Utils.createAceEditor('connectionLogDetail', 'ace/mode/json', true); + connectionStatusDetail = Utils.createAceEditor('connectionStatusDetail', 'ace/mode/json', true); + + // Status -------------- + dom.buttonRetrieveConnectionStatus.onclick = retrieveConnectionStatus; + document.querySelector('a[data-bs-target="#tabConnectionStatus"]').onclick = retrieveConnectionStatus; + + // Logs -------------- + dom.buttonEnableConnectionLogs.onclick = onEnableConnectionLogsClick; + dom.buttonResetConnectionLogs.onclick = onResetConnectionLogsClick; + dom.buttonRetrieveConnectionLogs.onclick = retrieveConnectionLogs; + dom.tbodyConnectionLogs.addEventListener('click', onConnectionLogTableClick); + + // Metrics --------------- + dom.buttonRetrieveConnectionMetrics.onclick = retrieveConnectionMetrics; + document.querySelector('a[data-bs-target="#tabConnectionMetrics"]').onclick = retrieveConnectionMetrics; + dom.buttonResetConnectionMetrics.onclick = onResetConnectionMetricsClick; +} + +function onResetConnectionMetricsClick() { + Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); + API.callConnectionsAPI('connectionCommand', retrieveConnectionMetrics, selectedConnectionId, null, 'connectivity.commands:resetConnectionMetrics'); +} + +function onConnectionLogTableClick(event) { + connectionLogDetail.setValue(JSON.stringify(connectionLogs[event.target.parentNode.rowIndex - 1], null, 2), -1); +} + +function onResetConnectionLogsClick() { + Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); + API.callConnectionsAPI('connectionCommand', retrieveConnectionLogs, selectedConnectionId, null, 'connectivity.commands:resetConnectionLogs'); +} + +function onEnableConnectionLogsClick() { + Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); + API.callConnectionsAPI('connectionCommand', retrieveConnectionLogs, selectedConnectionId, null, 'connectivity.commands:enableConnectionLogs'); +} + +function retrieveConnectionMetrics() { + Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); + dom.tbodyConnectionMetrics.innerHTML = ''; + API.callConnectionsAPI('retrieveConnectionMetrics', (response) => { + if (response.connectionMetrics) { + Object.keys(response.connectionMetrics).forEach((direction) => { + if (response.connectionMetrics[direction]) { + Object.keys(response.connectionMetrics[direction]).forEach((type) => { + let entry = response.connectionMetrics[direction][type]; + Utils.addTableRow(dom.tbodyConnectionMetrics, direction, false, false, type, 'success', entry.success.PT1M, entry.success.PT1H, entry.success.PT24H); + Utils.addTableRow(dom.tbodyConnectionMetrics, direction, false, false, type, 'failure', entry.failure.PT1M, entry.failure.PT1H, entry.failure.PT24H); + }); + }; + }); + } + }, + selectedConnectionId); +} + +function retrieveConnectionStatus() { + Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); + API.callConnectionsAPI('retrieveStatus', (connectionStatus) => { + connectionStatusDetail.setValue(JSON.stringify(connectionStatus, null, 2), -1); + }, + selectedConnectionId); +} + +function retrieveConnectionLogs() { + Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); + dom.tbodyConnectionLogs.innerHTML = ''; + connectionLogDetail.setValue(''); + API.callConnectionsAPI('retrieveConnectionLogs', (response) => { + connectionLogs = response.connectionLogs; + adjustEnableButton(response); + response.connectionLogs.forEach((entry) => { + Utils.addTableRow(dom.tbodyConnectionLogs, Utils.formatDate(entry.timestamp, true), false, false, entry.type, entry.level); + }); + dom.tbodyConnectionLogs.scrollTop = dom.tbodyConnectionLogs.scrollHeight - dom.tbodyConnectionLogs.clientHeight; + }, + selectedConnectionId); +} + +function adjustEnableButton(response) { + if (response.enabledUntil) { + dom.buttonEnableConnectionLogs.querySelector('i').classList.replace('bi-toggle-off', 'bi-toggle-on'); + dom.buttonEnableConnectionLogs.setAttribute('title', `Enabled until ${Utils.formatDate(response.enabledUntil)}`); + } else { + dom.buttonEnableConnectionLogs.querySelector('i').classList.replace('bi-toggle-on', 'bi-toggle-off'); + dom.buttonEnableConnectionLogs.setAttribute('title', 'Click to enable connection logs for the selected connection'); + } +} + +function onConnectionChange(connection, isNewConnection = true) { + selectedConnectionId = connection ? connection.id : null; + connectionStatusDetail.setValue(''); + connectionLogDetail.setValue(''); + dom.tbodyConnectionMetrics.innerHTML = ''; + dom.tbodyConnectionLogs.innerHTML = ''; + if (!isNewConnection && connection && connection.id) { + retrieveConnectionLogs(); + } +} diff --git a/ui/modules/things/attributes.js b/ui/modules/things/attributes.js index 1495193623f..ac2a0a085fa 100644 --- a/ui/modules/things/attributes.js +++ b/ui/modules/things/attributes.js @@ -154,7 +154,8 @@ function attributeFromString(attribute) { } function onEditToggle(event) { - if (event.detail) { + const isEditing = event.detail; + if (isEditing && dom.crudAttribute.idValue && dom.crudAttribute.idValue !== '') { API.callDittoREST('GET', `/things/${Things.theThing.thingId}/attributes/${dom.crudAttribute.idValue}`, null, null, true) .then((response) => { diff --git a/ui/modules/things/features.js b/ui/modules/things/features.js index e67fcf282f2..02350446e88 100644 --- a/ui/modules/things/features.js +++ b/ui/modules/things/features.js @@ -223,7 +223,8 @@ function onThingChanged(thing) { } function onEditToggle(event) { - if (event.detail) { + const isEditing = event.detail; + if (isEditing && dom.crudFeature.idValue && dom.crudFeature.idValue !== '') { API.callDittoREST('GET', `/things/${Things.theThing.thingId}/features/${dom.crudFeature.idValue}`, null, null, true) .then((response) => { eTag = response.headers.get('ETag'); @@ -240,11 +241,11 @@ function onEditToggle(event) { } function enableDisableEditors() { - dom.inputFeatureDefinition.disabled = !event.detail; - featurePropertiesEditor.setReadOnly(!event.detail); - featurePropertiesEditor.renderer.setShowGutter(event.detail); - featureDesiredPropertiesEditor.setReadOnly(!event.detail); - featureDesiredPropertiesEditor.renderer.setShowGutter(event.detail); + dom.inputFeatureDefinition.disabled = !isEditing; + featurePropertiesEditor.setReadOnly(!isEditing); + featurePropertiesEditor.renderer.setShowGutter(isEditing); + featureDesiredPropertiesEditor.setReadOnly(!isEditing); + featureDesiredPropertiesEditor.renderer.setShowGutter(isEditing); } function clearEditorsAfterCancel() { diff --git a/ui/modules/things/thingsCRUD.js b/ui/modules/things/thingsCRUD.js index 115ca1294d2..1cc1e7d8d64 100644 --- a/ui/modules/things/thingsCRUD.js +++ b/ui/modules/things/thingsCRUD.js @@ -163,7 +163,8 @@ function onThingChanged(thingJson) { } function onEditToggle(event) { - if (event.detail) { + const isEditing = event.detail; + if (isEditing && Things.theThing) { API.callDittoREST('GET', `/things/${Things.theThing.thingId}`, null, null, true) .then((response) => { eTag = response.headers.get('ETag'); @@ -179,14 +180,14 @@ function onEditToggle(event) { } function enableDisableEditors() { - dom.buttonThingDefinitions.disabled = !event.detail; - dom.inputThingDefinition.disabled = !event.detail; + dom.buttonThingDefinitions.disabled = !isEditing; + dom.inputThingDefinition.disabled = !isEditing; } function updateEditorsBeforeEdit(thingJson) { dom.inputThingDefinition.value = thingJson.definition ?? ''; - thingJsonEditor.setReadOnly(!event.detail); - thingJsonEditor.renderer.setShowGutter(event.detail); + thingJsonEditor.setReadOnly(!isEditing); + thingJsonEditor.renderer.setShowGutter(isEditing); thingJsonEditor.setValue(JSON.stringify(thingJson, null, 2), -1); } From 580c28077dd5f0f572ee4e6d159f13360f0f68af Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Fri, 13 Jan 2023 16:31:59 +0100 Subject: [PATCH 009/173] add additional status-secured configuration to enable disabling securing the status endpoint while still securing the devops endpoint Signed-off-by: Thomas Jaeckle --- deployment/docker/sandbox/docker-compose.yml | 2 +- .../DevopsAuthenticationDirectiveFactory.java | 2 +- .../config/security/DefaultDevOpsConfig.java | 23 +++++++++++++------ .../util/config/security/DevOpsConfig.java | 14 ++++++++++- .../service/src/main/resources/gateway.conf | 5 ++-- .../security/DefaultDevOpsConfigTest.java | 2 ++ .../src/test/resources/devops-test.conf | 1 + 7 files changed, 37 insertions(+), 12 deletions(-) diff --git a/deployment/docker/sandbox/docker-compose.yml b/deployment/docker/sandbox/docker-compose.yml index 11cc23f7103..684823b7352 100644 --- a/deployment/docker/sandbox/docker-compose.yml +++ b/deployment/docker/sandbox/docker-compose.yml @@ -135,7 +135,7 @@ services: - TZ=Europe/Berlin - BIND_HOSTNAME=0.0.0.0 - ENABLE_PRE_AUTHENTICATION=true - - DEVOPS_SECURE_STATUS=false + - DEVOPS_STATUS_SECURED=false - DITTO_DEVOPS_FEATURE_WOT_INTEGRATION_ENABLED=true - | JAVA_TOOL_OPTIONS= diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/directives/auth/DevopsAuthenticationDirectiveFactory.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/directives/auth/DevopsAuthenticationDirectiveFactory.java index 5a487b95537..19e0ec35f62 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/directives/auth/DevopsAuthenticationDirectiveFactory.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/directives/auth/DevopsAuthenticationDirectiveFactory.java @@ -40,7 +40,7 @@ public static DevopsAuthenticationDirectiveFactory newInstance( } public DevopsAuthenticationDirective status() { - if (!devOpsConfig.isSecured()) { + if (!devOpsConfig.isSecured() || !devOpsConfig.isStatusSecured()) { return DevOpsInsecureAuthenticationDirective.getInstance(); } switch (devOpsConfig.getStatusAuthenticationMethod()) { diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfig.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfig.java index e974d5a7a93..5e94e545a09 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfig.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfig.java @@ -33,23 +33,25 @@ public final class DefaultDevOpsConfig implements DevOpsConfig { private static final String CONFIG_PATH = "devops"; - private final boolean secureStatus; + private final boolean secured; private final DevopsAuthenticationMethod devopsAuthenticationMethod; private final String password; private final Collection devopsOAuth2Subjects; + private final boolean statusSecured; private final DevopsAuthenticationMethod statusAuthenticationMethod; private final String statusPassword; private final Collection statusOAuth2Subjects; private final OAuthConfig oAuthConfig; private DefaultDevOpsConfig(final ConfigWithFallback configWithFallback) { - secureStatus = configWithFallback.getBoolean(DevOpsConfigValue.SECURED.getConfigPath()); + secured = configWithFallback.getBoolean(DevOpsConfigValue.SECURED.getConfigPath()); devopsAuthenticationMethod = getDevopsAuthenticationMethod(configWithFallback, DevOpsConfigValue.DEVOPS_AUTHENTICATION_METHOD); password = configWithFallback.getString(DevOpsConfigValue.PASSWORD.getConfigPath()); devopsOAuth2Subjects = Collections.unmodifiableList(new ArrayList<>( configWithFallback.getStringList(DevOpsConfigValue.DEVOPS_OAUTH2_SUBJECTS.getConfigPath()))); + statusSecured = configWithFallback.getBoolean(DevOpsConfigValue.STATUS_SECURED.getConfigPath()); statusAuthenticationMethod = getDevopsAuthenticationMethod(configWithFallback, DevOpsConfigValue.STATUS_AUTHENTICATION_METHOD); statusPassword = configWithFallback.getString(DevOpsConfigValue.STATUS_PASSWORD.getConfigPath()); @@ -84,7 +86,7 @@ public static DefaultDevOpsConfig of(final Config config) { @Override public boolean isSecured() { - return secureStatus; + return secured; } @Override @@ -102,6 +104,11 @@ public Collection getDevopsOAuth2Subjects() { return devopsOAuth2Subjects; } + @Override + public boolean isStatusSecured() { + return statusSecured; + } + @Override public DevopsAuthenticationMethod getStatusAuthenticationMethod() { return statusAuthenticationMethod; @@ -131,10 +138,11 @@ public boolean equals(@Nullable final Object o) { return false; } final DefaultDevOpsConfig that = (DefaultDevOpsConfig) o; - return Objects.equals(secureStatus, that.secureStatus) && + return Objects.equals(secured, that.secured) && Objects.equals(devopsAuthenticationMethod, that.devopsAuthenticationMethod) && Objects.equals(password, that.password) && Objects.equals(devopsOAuth2Subjects, that.devopsOAuth2Subjects) && + Objects.equals(statusSecured, that.statusSecured) && Objects.equals(statusAuthenticationMethod, that.statusAuthenticationMethod) && Objects.equals(statusOAuth2Subjects, that.statusOAuth2Subjects) && Objects.equals(statusPassword, that.statusPassword) && @@ -143,17 +151,18 @@ public boolean equals(@Nullable final Object o) { @Override public int hashCode() { - return Objects.hash(secureStatus, devopsAuthenticationMethod, password, devopsOAuth2Subjects, - statusAuthenticationMethod, statusPassword, statusOAuth2Subjects, oAuthConfig); + return Objects.hash(secured, devopsAuthenticationMethod, password, devopsOAuth2Subjects, + statusSecured, statusAuthenticationMethod, statusPassword, statusOAuth2Subjects, oAuthConfig); } @Override public String toString() { return getClass().getSimpleName() + " [" + - "secureStatus=" + secureStatus + + "secured=" + secured + ", devopsAuthenticationMethod=" + devopsAuthenticationMethod + ", password=*****" + ", devopsOAuth2Subject=" + devopsOAuth2Subjects + + ", statusSecured=" + statusSecured + ", statusAuthenticationMethod=" + statusAuthenticationMethod + ", statusPassword=*****" + ", statusOAuth2Subject=" + statusOAuth2Subjects + diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DevOpsConfig.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DevOpsConfig.java index 0c23be7a711..59931d28192 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DevOpsConfig.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DevOpsConfig.java @@ -53,6 +53,13 @@ public interface DevOpsConfig { */ Collection getDevopsOAuth2Subjects(); + /** + * Indicates whether the DevOps resource {@code /status} should be secured or not. + * + * @return {@code true} if {@code /status} should be secured, {@code false} else; + */ + boolean isStatusSecured(); + /** * Returns the authentication method for status resources. * @@ -107,6 +114,11 @@ enum DevOpsConfigValue implements KnownConfigValue { */ DEVOPS_OAUTH2_SUBJECTS("devops-oauth2-subjects", List.of()), + /** + * Determines whether DevOps resource {@code /status} should be secured or not. + */ + STATUS_SECURED("status-secured", true), + /** * The authentication method for status resources. */ @@ -125,7 +137,7 @@ enum DevOpsConfigValue implements KnownConfigValue { private final String path; private final Object defaultValue; - private DevOpsConfigValue(final String thePath, final Object theDefaultValue) { + DevOpsConfigValue(final String thePath, final Object theDefaultValue) { path = thePath; defaultValue = theDefaultValue; } diff --git a/gateway/service/src/main/resources/gateway.conf b/gateway/service/src/main/resources/gateway.conf index e056311f1d1..f6e741fec8c 100755 --- a/gateway/service/src/main/resources/gateway.conf +++ b/gateway/service/src/main/resources/gateway.conf @@ -275,8 +275,6 @@ ditto { secured = true # Backwardcompatibility fallback secured = ${?ditto.gateway.authentication.devops.securestatus} - # Backwardcompatibility fallback - secured = ${?DEVOPS_SECURE_STATUS} secured = ${?DEVOPS_SECURED} # default authentiation method for the devops route @@ -296,6 +294,9 @@ ditto { # oauth2 auth devops-oauth2-subjects = ${?DEVOPS_OAUTH2_SUBJECTS} + status-secured = true + status-secured = ${?DEVOPS_STATUS_SECURED} + # default authentiation method for the status route # can be set to "basic" or "oauth" status-authentication-method = "basic" diff --git a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfigTest.java b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfigTest.java index 625a1be8054..ab867d08fd5 100644 --- a/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfigTest.java +++ b/gateway/service/src/test/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultDevOpsConfigTest.java @@ -65,6 +65,7 @@ public void underTestReturnsDefaultValuesIfBaseConfigWasEmpty() { DevOpsConfig.DevOpsConfigValue.DEVOPS_AUTHENTICATION_METHOD); assertDefaultValueFor(underTest.getPassword(), DevOpsConfig.DevOpsConfigValue.PASSWORD); assertDefaultValueFor(underTest.getDevopsOAuth2Subjects(), DevOpsConfig.DevOpsConfigValue.DEVOPS_OAUTH2_SUBJECTS); + assertDefaultValueFor(underTest.isStatusSecured(), DevOpsConfig.DevOpsConfigValue.STATUS_SECURED); assertDefaultValueFor(underTest.getStatusAuthenticationMethod().getMethodName(), DevOpsConfig.DevOpsConfigValue.STATUS_AUTHENTICATION_METHOD); assertDefaultValueFor(underTest.getStatusPassword(), DevOpsConfig.DevOpsConfigValue.STATUS_PASSWORD); @@ -87,6 +88,7 @@ public void underTestReturnsValuesOfConfigFile() { assertConfiguredValueFor(underTest.getPassword(), DevOpsConfig.DevOpsConfigValue.PASSWORD, "bumlux"); assertConfiguredValueFor(underTest.getDevopsOAuth2Subjects(), DevOpsConfig.DevOpsConfigValue.DEVOPS_OAUTH2_SUBJECTS, List.of("someissuer:a", "someissuer:b")); + assertConfiguredValueFor(underTest.isStatusSecured(), DevOpsConfig.DevOpsConfigValue.SECURED, false); assertConfiguredValueFor(underTest.getStatusAuthenticationMethod().getMethodName(), DevOpsConfig.DevOpsConfigValue.STATUS_AUTHENTICATION_METHOD, "oauth2"); assertConfiguredValueFor(underTest.getStatusPassword(), DevOpsConfig.DevOpsConfigValue.STATUS_PASSWORD, diff --git a/gateway/service/src/test/resources/devops-test.conf b/gateway/service/src/test/resources/devops-test.conf index 99edaf1b31e..d99ae282b42 100644 --- a/gateway/service/src/test/resources/devops-test.conf +++ b/gateway/service/src/test/resources/devops-test.conf @@ -3,6 +3,7 @@ devops { devops-authentication-method = "basic" password = "bumlux" devops-oauth2-subjects = ["someissuer:a","someissuer:b"] + status-secured = false status-authentication-method = "oauth2" statusPassword = "1234" status-oauth2-subjects = ["someissuer:c"] From 362390e5f561243cc6300c91e630a96e3364dc4c Mon Sep 17 00:00:00 2001 From: JeffreyThijs Date: Fri, 13 Jan 2023 17:01:20 +0100 Subject: [PATCH 010/173] external message to mqtt publish transform should not fail on blank header values --- ...ternalMessageToMqttPublishTransformer.java | 1 + ...alMessageToMqttPublishTransformerTest.java | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformer.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformer.java index ebc5ee326ea..65f84b46b3b 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformer.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformer.java @@ -236,6 +236,7 @@ private static Set getUserPropertiesOrEmptySet( ) { return externalMessageHeaders.stream() .filter(header -> !KNOWN_MQTT_HEADER_NAMES.contains(header.getKey())) + .filter(header -> header.getValue() != null && !header.getValue().isBlank()) .map(header -> { final var headerKey = header.getKey(); final String headerValue; diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformerTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformerTest.java index 7b10149aef7..b6c4ff97a77 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformerTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/publishing/ExternalMessageToMqttPublishTransformerTest.java @@ -276,4 +276,32 @@ public void transformExternalMessageWithInvalidRetainValueYieldsTransformationFa .hasCauseInstanceOf(InvalidHeaderValueException.class); } + @Test + public void transformFullyFledgedExternalMessageWithBlankHeaderReturnsExpectedTransformationSuccessResult() { + final var correlationId = testNameCorrelationId.getCorrelationId(); + final var genericMqttPublish = GenericMqttPublish.builder(MQTT_TOPIC, MQTT_QOS) + .retain(RETAIN) + .payload(PAYLOAD) + .correlationData(ByteBufferUtils.fromUtf8String(correlationId.toString())) + .contentType(CONTENT_TYPE.getValue()) + .responseTopic(REPLY_TO_TOPIC) + .userProperties(USER_PROPERTIES) + .build(); + Mockito.when(externalMessage.getHeaders()) + .thenReturn(DittoHeaders.newBuilder() + .putHeader(MqttHeader.MQTT_TOPIC.getName(), MQTT_TOPIC.toString()) + .putHeader(MqttHeader.MQTT_QOS.getName(), String.valueOf(MQTT_QOS.getCode())) + .putHeader(MqttHeader.MQTT_RETAIN.getName(), String.valueOf(genericMqttPublish.isRetain())) + .putHeader("ablankheader", "") + .correlationId(correlationId) + .putHeader(ExternalMessage.REPLY_TO_HEADER, REPLY_TO_TOPIC.toString()) + .contentType(CONTENT_TYPE) + .putHeaders(USER_PROPERTIES.stream() + .collect(Collectors.toMap(UserProperty::name, UserProperty::value))) + .build()); + Mockito.when(externalMessage.getBytePayload()).thenReturn(genericMqttPublish.getPayload()); + + assertThat(ExternalMessageToMqttPublishTransformer.transform(externalMessage, mqttPublishTarget)) + .isEqualTo(TransformationSuccess.of(externalMessage, genericMqttPublish)); + } } \ No newline at end of file From ab4f22e6fd58fb2b628037c16a325e801b7259a4 Mon Sep 17 00:00:00 2001 From: thfries Date: Sat, 14 Jan 2023 17:05:37 +0100 Subject: [PATCH 011/173] Explorere UI - SSE support - Added new view to see incoming updates for the selected thing - Utils format date had wrong interface description Signed-off-by: thfries --- ui/index.html | 1 + ui/main.js | 3 ++ ui/modules/things/messagesIncoming.html | 46 ++++++++++++++++ ui/modules/things/messagesIncoming.js | 72 +++++++++++++++++++++++++ ui/modules/things/thingsSSE.js | 6 +++ ui/modules/utils.js | 10 ++-- 6 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 ui/modules/things/messagesIncoming.html create mode 100644 ui/modules/things/messagesIncoming.js diff --git a/ui/index.html b/ui/index.html index 83ba6a5a44b..38798e1dad0 100644 --- a/ui/index.html +++ b/ui/index.html @@ -96,6 +96,7 @@
+
diff --git a/ui/main.js b/ui/main.js index 598b9983bd3..779b9eb4c51 100644 --- a/ui/main.js +++ b/ui/main.js @@ -23,6 +23,7 @@ import * as Things from './modules/things/things.js'; import * as ThingsSearch from './modules/things/thingsSearch.js'; import * as ThingsCRUD from './modules/things/thingsCRUD.js'; import * as ThingsSSE from './modules/things/thingsSSE.js'; +import * as MessagesIncoming from './modules/things/messagesIncoming.js'; import * as Connections from './modules/connections/connections.js'; import * as ConnectionsCRUD from './modules/connections/connectionsCRUD.js'; import * as ConnectionsMonitor from './modules/connections/connectionsMonitor.js'; @@ -39,6 +40,7 @@ document.addEventListener('DOMContentLoaded', async function() { document.getElementById('thingsHTML').innerHTML = await (await fetch('modules/things/things.html')).text(); document.getElementById('fieldsHTML').innerHTML = await (await fetch('modules/things/fields.html')).text(); document.getElementById('featuresHTML').innerHTML = await (await fetch('modules/things/features.html')).text(); + document.getElementById('messagesIncomingHTML').innerHTML = await (await fetch('modules/things/messagesIncoming.html')).text(); document.getElementById('policyHTML').innerHTML = await (await fetch('modules/policies/policies.html')).text(); document.getElementById('connectionsHTML').innerHTML = await (await fetch('modules/connections/connections.html')).text(); @@ -52,6 +54,7 @@ document.addEventListener('DOMContentLoaded', async function() { ThingsSearch.ready(); ThingsCRUD.ready(); ThingsSSE.ready(); + MessagesIncoming.ready(); Attributes.ready(); await Fields.ready(); await SearchFilter.ready(); diff --git a/ui/modules/things/messagesIncoming.html b/ui/modules/things/messagesIncoming.html new file mode 100644 index 00000000000..998d716ccb8 --- /dev/null +++ b/ui/modules/things/messagesIncoming.html @@ -0,0 +1,46 @@ + +
+ Incoming Thing Updates
+
+
+
+
+
+ +
+
+ + + + + + + + + +
RevisionFeatureIdModified
+
+
+
+
Thing Update Detail
+
+
+
+
+
+
\ No newline at end of file diff --git a/ui/modules/things/messagesIncoming.js b/ui/modules/things/messagesIncoming.js new file mode 100644 index 00000000000..b1033b80591 --- /dev/null +++ b/ui/modules/things/messagesIncoming.js @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +import * as Utils from '../utils.js'; +import * as Things from './things.js'; +import * as ThingsSSE from './thingsSSE.js'; +/* eslint-disable prefer-const */ +/* eslint-disable max-len */ +/* eslint-disable no-invalid-this */ +/* eslint-disable require-jsdoc */ + +let dom = { + badgeMessageIncomingCount: null, + buttonResetMessagesIncoming: null, + tbodyMessagesIncoming: null, +}; + +let messages = []; +let messageDetail; +let currentThingId; + +export function ready() { + ThingsSSE.setObserver(onMessage); + Things.addChangeListener(onThingChanged); + + Utils.getAllElementsById(dom); + + messageDetail = Utils.createAceEditor('messageIncomingDetail', 'ace/mode/json', true); + + dom.buttonResetMessagesIncoming.onclick = onResetMessagesClick; + dom.tbodyMessagesIncoming.addEventListener('click', onMessageTableClick); +} + +function onMessageTableClick(event) { + messageDetail.setValue(JSON.stringify(messages[event.target.parentNode.rowIndex - 1], null, 2), -1); +} + +function onResetMessagesClick() { + messages = []; + dom.badgeMessageIncomingCount.textContent = ''; + dom.tbodyMessagesIncoming.innerHTML = ''; + messageDetail.setValue(''); +} + +function onMessage(messageData) { + messages.push(messageData); + dom.badgeMessageIncomingCount.textContent = messages.length; + + Utils.addTableRow( + dom.tbodyMessagesIncoming, + messageData._revision, false, false, + messageData['features'] ? Object.keys(messageData.features) : '', + Utils.formatDate(messageData._modified, true), + ); +} + +function onThingChanged(thing) { + if (!thing || thing.thingId !== currentThingId) { + currentThingId = thing ? thing.thingId : null; + onResetMessagesClick(); + } +} diff --git a/ui/modules/things/thingsSSE.js b/ui/modules/things/thingsSSE.js index 817bc9f0c94..0448be1207a 100644 --- a/ui/modules/things/thingsSSE.js +++ b/ui/modules/things/thingsSSE.js @@ -26,6 +26,11 @@ export async function ready() { Things.addChangeListener(onThingChanged); } +let observer; +export function setObserver(newObserver) { + observer = newObserver; +} + function onThingChanged(newThingJson, isNewThingId) { if (!newThingJson) { thingEventSource && thingEventSource.close(); @@ -44,6 +49,7 @@ function onMessage(event) { console.log(event); const merged = _.merge(Things.theThing, JSON.parse(event.data)); Things.setTheThing(merged); + observer && observer.call(null, JSON.parse(event.data)); } } diff --git a/ui/modules/utils.js b/ui/modules/utils.js index 2cbb98424ab..8ed6f86505b 100644 --- a/ui/modules/utils.js +++ b/ui/modules/utils.js @@ -259,16 +259,16 @@ export function assert(condition, message, validatedElement) { } /** - * Simple Date format that makes UTC string more readable and cuts off the milliseconds - * @param {Date} date to format + * Simple Date format that makes ISO string more readable and cuts off the milliseconds + * @param {String} dateISOString to format * @param {boolean} withMilliseconds don t cut off milliseconds if true * @return {String} formatted date */ -export function formatDate(date, withMilliseconds) { +export function formatDate(dateISOString, withMilliseconds) { if (withMilliseconds) { - return date.replace('T', ' ').replace('Z', '').replace('.', ' '); + return dateISOString.replace('T', ' ').replace('Z', '').replace('.', ' '); } else { - return date.split('.')[0].replace('T', ' '); + return dateISOString.split('.')[0].replace('T', ' '); } } From 9cd1539319445e4d72e1bb13d5081f74fa617ffb Mon Sep 17 00:00:00 2001 From: thfries Date: Sun, 15 Jan 2023 07:36:25 +0100 Subject: [PATCH 012/173] Explorer UI - SSE support - On feature update also weak ETags are allowed - Improved cleaning up on environment change Signed-off-by: thfries --- ui/modules/things/features.js | 2 +- ui/modules/things/thingsSSE.js | 17 +++++++++++++++-- ui/modules/things/thingsSearch.js | 23 +++++++++++++++++------ ui/modules/things/wotDescription.js | 2 +- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/ui/modules/things/features.js b/ui/modules/things/features.js index 02350446e88..84646bab70a 100644 --- a/ui/modules/things/features.js +++ b/ui/modules/things/features.js @@ -227,7 +227,7 @@ function onEditToggle(event) { if (isEditing && dom.crudFeature.idValue && dom.crudFeature.idValue !== '') { API.callDittoREST('GET', `/things/${Things.theThing.thingId}/features/${dom.crudFeature.idValue}`, null, null, true) .then((response) => { - eTag = response.headers.get('ETag'); + eTag = response.headers.get('ETag').replace('W/', ''); return response.json(); }) .then((featureJson) => { diff --git a/ui/modules/things/thingsSSE.js b/ui/modules/things/thingsSSE.js index 0448be1207a..a6cf897e70e 100644 --- a/ui/modules/things/thingsSSE.js +++ b/ui/modules/things/thingsSSE.js @@ -13,6 +13,7 @@ /* eslint-disable require-jsdoc */ // @ts-check import * as API from '../api.js'; +import * as Environments from '../environments/environments.js'; import * as Things from './things.js'; @@ -24,6 +25,7 @@ let thingEventSource; */ export async function ready() { Things.addChangeListener(onThingChanged); + Environments.addChangeListener(onEnvironmentChanged); } let observer; @@ -33,8 +35,7 @@ export function setObserver(newObserver) { function onThingChanged(newThingJson, isNewThingId) { if (!newThingJson) { - thingEventSource && thingEventSource.close(); - thingEventSource = null; + stopSSE(); } else if (isNewThingId) { thingEventSource && thingEventSource.close(); console.log('Start SSE: ' + newThingJson.thingId); @@ -44,6 +45,18 @@ function onThingChanged(newThingJson, isNewThingId) { } } +function stopSSE() { + thingEventSource && thingEventSource.close(); + thingEventSource = null; + console.log('SSE Stopped'); +} + +function onEnvironmentChanged(modifiedField) { + if (!['pinnedThings', 'filterList', 'messageTemplates'].includes(modifiedField)) { + stopSSE(); + } +} + function onMessage(event) { if (event.data && event.data !== '') { console.log(event); diff --git a/ui/modules/things/thingsSearch.js b/ui/modules/things/thingsSearch.js index 3aa3c7d87c7..5d02708891c 100644 --- a/ui/modules/things/thingsSearch.js +++ b/ui/modules/things/thingsSearch.js @@ -107,8 +107,21 @@ export function getThings(thingIds) { dom.thingsTableBody.innerHTML = ''; if (thingIds.length > 0) { API.callDittoREST('GET', - `/things?${Fields.getQueryParameter()}&ids=${thingIds}&option=sort(%2BthingId)`, - ).then(fillThingsTable); + `/things?${Fields.getQueryParameter()}&ids=${thingIds}&option=sort(%2BthingId)`) + .then(fillThingsTable) + .catch((error) => { + resetAndClearViews(); + }); + } else { + resetAndClearViews(); + } +} + +function resetAndClearViews(retainThing = false) { + theSearchCursor = null; + dom.thingsTableBody.innerHTML = ''; + if (!retainThing) { + Things.setTheThing(null); } } @@ -133,14 +146,12 @@ function searchThings(filter, isMore = false) { if (isMore) { removeMoreFromThingList(); } else { - theSearchCursor = null; - dom.thingsTableBody.innerHTML = ''; + resetAndClearViews(true); } fillThingsTable(searchResult.items); checkMorePages(searchResult); }).catch((error) => { - theSearchCursor = null; - dom.thingsTableBody.innerHTML = ''; + resetAndClearViews(); }).finally(() => { document.body.style.cursor = 'default'; }); diff --git a/ui/modules/things/wotDescription.js b/ui/modules/things/wotDescription.js index 7e099772b75..77032d652b0 100644 --- a/ui/modules/things/wotDescription.js +++ b/ui/modules/things/wotDescription.js @@ -44,7 +44,7 @@ export function WoTDescription(targetTab, forFeature) { }; const onReferenceChanged = () => { - if (tabLink.classList.contains('active')) { + if (tabLink && tabLink.classList.contains('active')) { refreshDescription(); } else { viewDirty = true; From 4b37709f8efd5b58379fb4c71b9432bdeb86b987 Mon Sep 17 00:00:00 2001 From: thfries Date: Sun, 15 Jan 2023 21:40:02 +0100 Subject: [PATCH 013/173] Explorer UI - SSE support - Changed CRUD buttons to text and showing dynamically - wrong ETag header on feature/attribute creation - incoming changes now also show attributes Signed-off-by: thfries --- ui/index.html | 20 ++++++----- ui/modules/things/attributes.js | 43 +++++++++++++---------- ui/modules/things/features.js | 15 +++++--- ui/modules/things/messagesIncoming.html | 2 +- ui/modules/things/messagesIncoming.js | 3 +- ui/modules/utils/crudToolbar.js | 46 +++++++++++++++++++------ 6 files changed, 86 insertions(+), 43 deletions(-) diff --git a/ui/index.html b/ui/index.html index 38798e1dad0..d9d9935f995 100644 --- a/ui/index.html +++ b/ui/index.html @@ -146,21 +146,25 @@
- - - - +
diff --git a/ui/modules/things/attributes.js b/ui/modules/things/attributes.js index ac2a0a085fa..d851e73a56a 100644 --- a/ui/modules/things/attributes.js +++ b/ui/modules/things/attributes.js @@ -1,16 +1,17 @@ -/* eslint-disable require-jsdoc */ /* - * Copyright (c) 2022 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ +* Copyright (c) 2022 Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Eclipse Public License 2.0 which is available at +* http://www.eclipse.org/legal/epl-2.0 +* +* SPDX-License-Identifier: EPL-2.0 +*/ +/* eslint-disable require-jsdoc */ +/* eslint-disable comma-dangle */ import * as API from '../api.js'; import * as Utils from '../utils.js'; @@ -48,7 +49,7 @@ function onCreateAttributeClick() { dom.crudAttribute.validationElement); Utils.assert(dom.inputAttributeValue.value, 'Attribute value must not be empty', dom.inputAttributeValue); - updateAttribute('PUT'); + updateAttribute('PUT', true); } function onUpdateAttributeClick() { Utils.assert(dom.inputAttributeValue.value, 'Attribute value must not be empty'); @@ -75,14 +76,19 @@ function onAttributeTableClick(event) { /** * Creates a onclick handler function * @param {String} method PUT or DELETE + * @param {boolean} isNewAttribute if a new attribute is created. default = false */ -function updateAttribute(method) { +function updateAttribute(method, isNewAttribute) { API.callDittoREST( method, `/things/${Things.theThing.thingId}/attributes/${dom.crudAttribute.idValue}`, method === 'PUT' ? attributeFromString(dom.inputAttributeValue.value) : null, + isNewAttribute ? + { + 'If-None-Match': '*' + } : { - 'if-match': method === 'PUT' ? eTag : '*', + 'If-Match': method === 'PUT' ? eTag : '*' }, ).then(() => { if (method === 'PUT') { @@ -155,6 +161,7 @@ function attributeFromString(attribute) { function onEditToggle(event) { const isEditing = event.detail; + dom.inputAttributeValue.disabled = !isEditing; if (isEditing && dom.crudAttribute.idValue && dom.crudAttribute.idValue !== '') { API.callDittoREST('GET', `/things/${Things.theThing.thingId}/attributes/${dom.crudAttribute.idValue}`, null, null, true) @@ -163,11 +170,11 @@ function onEditToggle(event) { return response.json(); }) .then((attributeValue) => { - dom.inputAttributeValue.disabled = false; dom.inputAttributeValue.value = attributeToString(attributeValue); }); } else { - dom.inputAttributeValue.disabled = true; - dom.inputAttributeValue.value = attributeToString(Things.theThing.attributes[dom.crudAttribute.idValue]); + dom.inputAttributeValue.value = dom.crudAttribute.idValue ? + attributeToString(Things.theThing.attributes[dom.crudAttribute.idValue]) : + null; } } diff --git a/ui/modules/things/features.js b/ui/modules/things/features.js index 84646bab70a..5d5d813965d 100644 --- a/ui/modules/things/features.js +++ b/ui/modules/things/features.js @@ -106,7 +106,7 @@ function onCreateFeatureClick() { Utils.assert(!Things.theThing['features'] || !Object.keys(Things.theThing.features).includes(dom.crudFeature.idValue), `Feature ID ${dom.crudFeature.idValue} already exists in Thing`, dom.crudFeature.validationElement); - updateFeature('PUT'); + updateFeature('PUT', true); } function onFeaturesTableClick(event) { @@ -122,8 +122,9 @@ function onFeaturesTableClick(event) { /** * Triggers a feature update in Ditto according to UI contents * @param {String} method Either PUT to update the feature or DELETE to delete the feature + * @param {boolean} isNewFeature indicates if a new feature should be created. (default: false) */ -function updateFeature(method) { +function updateFeature(method, isNewFeature = false) { Utils.assert(Things.theThing, 'No Thing selected'); Utils.assert(dom.crudFeature.idValue, 'No Feature selected'); @@ -144,8 +145,12 @@ function updateFeature(method) { method, '/things/' + Things.theThing.thingId + '/features/' + dom.crudFeature.idValue, method === 'PUT' ? featureObject : null, + isNewFeature ? + { + 'If-None-Match': '*' + } : { - 'if-match': method === 'PUT' ? eTag : '*' + 'If-Match': method === 'PUT' ? eTag : '*' } ).then(() => { if (method === 'PUT') { @@ -236,7 +241,7 @@ function onEditToggle(event) { }); } else { enableDisableEditors(); - clearEditorsAfterCancel(); + resetEditors(); dom.crudFeature.validationElement.classList.remove('is-invalid'); } @@ -248,7 +253,7 @@ function onEditToggle(event) { featureDesiredPropertiesEditor.renderer.setShowGutter(isEditing); } - function clearEditorsAfterCancel() { + function resetEditors() { if (dom.crudFeature.idValue) { refreshFeature(Things.theThing, dom.crudFeature.idValue); } else { diff --git a/ui/modules/things/messagesIncoming.html b/ui/modules/things/messagesIncoming.html index 998d716ccb8..5dd3a86b673 100644 --- a/ui/modules/things/messagesIncoming.html +++ b/ui/modules/things/messagesIncoming.html @@ -28,7 +28,7 @@
Revision - FeatureId + Field Modified diff --git a/ui/modules/things/messagesIncoming.js b/ui/modules/things/messagesIncoming.js index b1033b80591..dd369515433 100644 --- a/ui/modules/things/messagesIncoming.js +++ b/ui/modules/things/messagesIncoming.js @@ -59,7 +59,8 @@ function onMessage(messageData) { Utils.addTableRow( dom.tbodyMessagesIncoming, messageData._revision, false, false, - messageData['features'] ? Object.keys(messageData.features) : '', + [...messageData['features'] ? Object.keys(messageData.features) : [], + ...messageData['attributes'] ? Object.keys(messageData.attributes) : []], Utils.formatDate(messageData._modified, true), ); } diff --git a/ui/modules/utils/crudToolbar.js b/ui/modules/utils/crudToolbar.js index 48cfda086cd..8ac04543bdd 100644 --- a/ui/modules/utils/crudToolbar.js +++ b/ui/modules/utils/crudToolbar.js @@ -15,6 +15,7 @@ import * as Utils from '../utils.js'; class CrudToolbar extends HTMLElement { isEditing = false; + isEditDisabled = false; dom = { label: null, inputIdValue: null, @@ -22,7 +23,7 @@ class CrudToolbar extends HTMLElement { buttonCreate: null, buttonUpdate: null, buttonDelete: null, - iEdit: null, + buttonCancel: null, divRoot: null, }; @@ -32,15 +33,26 @@ class CrudToolbar extends HTMLElement { set idValue(newValue) { this.dom.inputIdValue.value = newValue; - this.dom.buttonDelete.disabled = (newValue === null); + if (newValue && newValue !== '') { + this.dom.buttonDelete.removeAttribute('hidden'); + } else { + this.dom.buttonDelete.setAttribute('hidden', ''); + } } get editDisabled() { - return this.dom.buttonCrudEdit.disabled; + return this.isEditDisabled; } set editDisabled(newValue) { - this.dom.buttonCrudEdit.disabled = newValue; + this.isEditDisabled = newValue; + if (!this.isEditing) { + if (this.isEditDisabled) { + this.dom.buttonCrudEdit.setAttribute('hidden', ''); + } else { + this.dom.buttonCrudEdit.removeAttribute('hidden'); + } + } } get validationElement() { @@ -57,6 +69,7 @@ class CrudToolbar extends HTMLElement { setTimeout(() => { Utils.getAllElementsById(this.dom, this.shadowRoot); this.dom.buttonCrudEdit.onclick = () => this.toggleEdit(); + this.dom.buttonCancel.onclick = () => this.toggleEdit(); this.dom.label.innerText = this.getAttribute('label') || 'Label'; this.dom.buttonCreate.onclick = this.eventDispatcher('onCreateClick'); this.dom.buttonUpdate.onclick = this.eventDispatcher('onUpdateClick'); @@ -76,13 +89,26 @@ class CrudToolbar extends HTMLElement { this.isEditing = !this.isEditing; document.getElementById('modalCrudEdit').classList.toggle('editBackground'); this.dom.divRoot.classList.toggle('editForground'); - this.dom.iEdit.classList.toggle('bi-pencil-square'); - this.dom.iEdit.classList.toggle('bi-x-square'); - this.dom.buttonCrudEdit.title = this.isEditing ? 'Cancel' : 'Edit'; - this.dom.buttonCreate.disabled = !(this.isEditing && !this.dom.inputIdValue.value); - this.dom.buttonUpdate.disabled = !(this.isEditing && this.dom.inputIdValue.value); + + if (this.isEditing || this.isEditDisabled) { + this.dom.buttonCrudEdit.setAttribute('hidden', ''); + } else { + this.dom.buttonCrudEdit.removeAttribute('hidden'); + } + this.dom.buttonCancel.toggleAttribute('hidden'); + + if (this.isEditing) { + if (this.dom.inputIdValue.value) { + this.dom.buttonUpdate.toggleAttribute('hidden'); + } else { + this.dom.buttonCreate.toggleAttribute('hidden'); + } + } else { + this.dom.buttonCreate.setAttribute('hidden', ''); + this.dom.buttonUpdate.setAttribute('hidden', ''); + } if (this.isEditing || !this.dom.inputIdValue.value) { - this.dom.buttonDelete.disabled = true; + this.dom.buttonDelete.setAttribute('hidden', ''); } const allowIdChange = this.isEditing && (!this.dom.inputIdValue.value || this.hasAttribute('allowIdChange')); this.dom.inputIdValue.disabled = !allowIdChange; From 0c7f7494d4af446fbd38825c23c65f7b091c1d42 Mon Sep 17 00:00:00 2001 From: thfries Date: Mon, 16 Jan 2023 12:44:19 +0100 Subject: [PATCH 014/173] Explorer UI - SSE bugfix Signed-off-by: thfries --- ui/modules/things/thingsCRUD.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/modules/things/thingsCRUD.js b/ui/modules/things/thingsCRUD.js index 1cc1e7d8d64..93319eed8b8 100644 --- a/ui/modules/things/thingsCRUD.js +++ b/ui/modules/things/thingsCRUD.js @@ -172,26 +172,26 @@ function onEditToggle(event) { }) .then((thingJson) => { enableDisableEditors(); - updateEditorsBeforeEdit(thingJson); + initializeEditors(thingJson); }); } else { enableDisableEditors(); - clearEditorsAfterCancel(); + resetEditors(); } function enableDisableEditors() { dom.buttonThingDefinitions.disabled = !isEditing; dom.inputThingDefinition.disabled = !isEditing; + thingJsonEditor.setReadOnly(!isEditing); + thingJsonEditor.renderer.setShowGutter(isEditing); } - function updateEditorsBeforeEdit(thingJson) { + function initializeEditors(thingJson) { dom.inputThingDefinition.value = thingJson.definition ?? ''; - thingJsonEditor.setReadOnly(!isEditing); - thingJsonEditor.renderer.setShowGutter(isEditing); thingJsonEditor.setValue(JSON.stringify(thingJson, null, 2), -1); } - function clearEditorsAfterCancel() { + function resetEditors() { if (dom.crudThings.idValue && dom.crudThings.idValue !== '') { Things.refreshThing(dom.crudThings.idValue, null); } else { From d7f78c295f73aa0a41cd0b44735fecf9a203d619 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Tue, 17 Jan 2023 10:13:05 +0100 Subject: [PATCH 015/173] added documentation about enhancing the JS payload mapping with custom libraries Signed-off-by: Thomas Jaeckle --- .../pages/ditto/connectivity-mapping.md | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/documentation/src/main/resources/pages/ditto/connectivity-mapping.md b/documentation/src/main/resources/pages/ditto/connectivity-mapping.md index ae108d3435f..d03d3af5c7b 100644 --- a/documentation/src/main/resources/pages/ditto/connectivity-mapping.md +++ b/documentation/src/main/resources/pages/ditto/connectivity-mapping.md @@ -546,6 +546,29 @@ In order to work more conveniently with binary payloads, the following libraries * [long.js](https://github.com/dcodeIO/long.js) for representing a 64-bit two's-complement integer value +### Adding additional JS libraries + +The used [Rhino JS engine](https://github.com/mozilla/rhino) allows making use of "CommonJS" in order to load JS +modules via `require('')` into the engine. +This feature is exposed to Ditto, configuring the configuration key `commonJsModulePath` or environment variable +`CONNECTIVITY_MESSAGE_MAPPING_JS_COMMON_JS_MODULE_PATH` of the connectivity service to a path in the +connectivity Docker container where to load additional CommonJS modules from - e.g. use a volume mount in order to get +additional JS modules into the container. + +For example, configure this variable to a folder to which you add (our mount) JavaScript libraries: +``` +CONNECTIVITY_MESSAGE_MAPPING_JS_COMMON_JS_MODULE_PATH=/opt/commonjs-modules/ +``` + +Then, for example, put [`pbf.js`](https://www.npmjs.com/package/pbf) (or any other JS library you want to use) +into that folder. + +Afterwards, the library can be used in your JS snippet using: +```javascript +var Pbf = require('pbf'); +``` + + ### Helper functions Ditto comes with a few helper functions, which makes writing the mapping scripts easier. They are available under the @@ -1214,7 +1237,8 @@ public final class FooMapper extends AbstractMessageMapper { } @Override - protected void doConfigure(Connection connection, MappingConfig mappingConfig, MessageMapperConfiguration configuration) { + private void doConfigure(Connection connection, MappingConfig mappingConfig, + MessageMapperConfiguration configuration) { // extract configuration if needed } } From 569e76cc2a0347880f8da376e3676746a85e8fc0 Mon Sep 17 00:00:00 2001 From: thfries Date: Thu, 19 Jan 2023 07:35:53 +0100 Subject: [PATCH 016/173] UI - SSE support: - Update selected thing in search result table Signed-off-by: thfries --- ui/modules/things/messagesIncoming.js | 2 +- ui/modules/things/thingsSSE.js | 12 ++++++++---- ui/modules/things/thingsSearch.js | 20 +++++++++++++++++++- ui/modules/utils.js | 2 ++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/ui/modules/things/messagesIncoming.js b/ui/modules/things/messagesIncoming.js index dd369515433..f9b0341aba9 100644 --- a/ui/modules/things/messagesIncoming.js +++ b/ui/modules/things/messagesIncoming.js @@ -30,7 +30,7 @@ let messageDetail; let currentThingId; export function ready() { - ThingsSSE.setObserver(onMessage); + ThingsSSE.addChangeListener(onMessage); Things.addChangeListener(onThingChanged); Utils.getAllElementsById(dom); diff --git a/ui/modules/things/thingsSSE.js b/ui/modules/things/thingsSSE.js index a6cf897e70e..ed46ac5c6c0 100644 --- a/ui/modules/things/thingsSSE.js +++ b/ui/modules/things/thingsSSE.js @@ -11,6 +11,7 @@ * SPDX-License-Identifier: EPL-2.0 */ /* eslint-disable require-jsdoc */ +/* eslint-disable arrow-parens */ // @ts-check import * as API from '../api.js'; import * as Environments from '../environments/environments.js'; @@ -28,9 +29,12 @@ export async function ready() { Environments.addChangeListener(onEnvironmentChanged); } -let observer; -export function setObserver(newObserver) { - observer = newObserver; +const observers = []; +export function addChangeListener(observer) { + observers.push(observer); +} +function notifyAll(thingJson) { + observers.forEach(observer => observer.call(null, thingJson)); } function onThingChanged(newThingJson, isNewThingId) { @@ -62,7 +66,7 @@ function onMessage(event) { console.log(event); const merged = _.merge(Things.theThing, JSON.parse(event.data)); Things.setTheThing(merged); - observer && observer.call(null, JSON.parse(event.data)); + notifyAll(JSON.parse(event.data)); } } diff --git a/ui/modules/things/thingsSearch.js b/ui/modules/things/thingsSearch.js index 5d02708891c..7ca6f0b8787 100644 --- a/ui/modules/things/thingsSearch.js +++ b/ui/modules/things/thingsSearch.js @@ -25,6 +25,7 @@ import * as API from '../api.js'; import * as Utils from '../utils.js'; import * as Fields from './fields.js'; import * as Things from './things.js'; +import * as ThingsSSE from './thingsSSE.js'; import * as Environments from '../environments/environments.js'; let lastSearch = ''; @@ -39,6 +40,7 @@ const dom = { export async function ready() { Things.addChangeListener(onThingChanged); + ThingsSSE.addChangeListener(onSelectedThingUpdate); Utils.getAllElementsById(dom); @@ -236,7 +238,7 @@ function fillThingsTable(thingsList) { json: item, path: path, }); - Utils.addCellToRow(row, elem.length !== 0 ? elem[0] : ''); + Utils.addCellToRow(row, elem.length !== 0 ? elem[0] : '').setAttribute('jsonPath', path); }); } @@ -270,3 +272,19 @@ function onThingChanged(thingJson) { Utils.tableAdjustSelection(dom.thingsTableBody, () => false); } } + +function onSelectedThingUpdate(thingUpdateJson) { + const row = document.getElementById(thingUpdateJson.thingId); + Array.from(row.cells).forEach((cell) => { + const path = cell.getAttribute('jsonPath'); + if (path) { + const elem = JSONPath({ + json: thingUpdateJson, + path: path, + }); + if (elem.length !== 0) { + cell.innerHTML = elem[0]; + } + } + }); +} diff --git a/ui/modules/utils.js b/ui/modules/utils.js index 8ed6f86505b..af9e595832a 100644 --- a/ui/modules/utils.js +++ b/ui/modules/utils.js @@ -73,12 +73,14 @@ export function addCheckboxToRow(row, id, checked, onToggle) { * @param {String} cellContent content of new cell * @param {String} cellTooltip tooltip for new cell * @param {Number} position optional, default -1 (add to the end) + * @return {HTMLElement} created cell element */ export function addCellToRow(row, cellContent, cellTooltip = null, position = -1) { const cell = row.insertCell(position); cell.innerHTML = cellContent; cell.setAttribute('data-bs-toggle', 'tooltip'); cell.title = cellTooltip ?? cellContent; + return cell; } /** From 298bf2da4aaf0a9c151f1aa170c24c792effbc93 Mon Sep 17 00:00:00 2001 From: Stanchev Aleksandar Date: Thu, 19 Jan 2023 17:08:12 +0200 Subject: [PATCH 017/173] extracts rawUserInfo to prevent unwanted url decoding Signed-off-by: Stanchev Aleksandar --- .../service/messaging/persistence/JsonFieldsEncryptor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/JsonFieldsEncryptor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/JsonFieldsEncryptor.java index a1d8a9ac4f6..fc617f226e8 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/JsonFieldsEncryptor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/JsonFieldsEncryptor.java @@ -87,7 +87,7 @@ public static JsonObject decrypt(final JsonObject jsonObject, final String point } static String replaceUriPassword(final String uriStringRepresentation, final String patchedPassword) { - final String userInfo = URI.create(uriStringRepresentation).getUserInfo(); + final String userInfo = URI.create(uriStringRepresentation).getRawUserInfo(); final String newUserInfo = userInfo.substring(0, userInfo.indexOf(":") + 1) + patchedPassword; final int startOfPwd = uriStringRepresentation.indexOf(userInfo); final int endOfPassword = uriStringRepresentation.indexOf("@"); @@ -159,7 +159,7 @@ private static Optional getUriPassword(final String uriStringRepresentat .message("Not a valid connection URI") .build(); } - final String userInfo = uri.getUserInfo(); + final String userInfo = uri.getRawUserInfo(); if (userInfo == null) { return Optional.empty(); } From 694b22b1f86cc126f57a42661a7d09ab7b0694a0 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Fri, 20 Jan 2023 10:31:22 +0100 Subject: [PATCH 018/173] Update SECURITY.md --- SECURITY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SECURITY.md b/SECURITY.md index db6acad65fa..4992edf5be1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,7 +1,10 @@ # Security Policy +Eclipse Ditto follows the [Eclipse Vulnerability Reporting Policy](https://www.eclipse.org/security/policy.php). Vulnerabilities are tracked by the Eclipse security team, in cooperation with the Ditto project leads. Fixing vulnerabilities is taken care of by the Ditto project committers, with assistance and guidance of the security team. + ## Supported Versions +Eclipse Ditto provides security updates for the two most recent minor versions. These versions of Eclipse Ditto are currently being supported with security updates. | Version | Supported | From 54dc02359b4cca0bce3497101b010f2c754ed63c Mon Sep 17 00:00:00 2001 From: thfries Date: Sat, 21 Jan 2023 22:08:34 +0100 Subject: [PATCH 019/173] UI - SSE Support: - create a second SSE that listens to all things of the search result - avoid ace editors from creating endless undo histories --- ui/modules/api.js | 4 +- ui/modules/connections/connectionsMonitor.js | 2 + ui/modules/things/features.js | 2 + ui/modules/things/messagesIncoming.js | 1 + ui/modules/things/thingsCRUD.js | 2 + ui/modules/things/thingsSSE.js | 51 ++++++++++++++------ ui/modules/things/thingsSearch.js | 31 +++++++++--- 7 files changed, 69 insertions(+), 24 deletions(-) diff --git a/ui/modules/api.js b/ui/modules/api.js index 95b3853aa9f..cdadd62a39e 100644 --- a/ui/modules/api.js +++ b/ui/modules/api.js @@ -338,9 +338,9 @@ export async function callDittoREST(method, path, body, additionalHeaders, retur } } -export function getEventSource(thingId, urlParams) { +export function getEventSource(thingIds, urlParams) { return new EventSourcePolyfill( - `${Environments.current().api_uri}/api/2/things?ids=${thingId}${urlParams ?? ''}`, { + `${Environments.current().api_uri}/api/2/things?ids=${thingIds}${urlParams ? '&' + urlParams : ''}`, { headers: { [authHeaderKey]: authHeaderValue, }, diff --git a/ui/modules/connections/connectionsMonitor.js b/ui/modules/connections/connectionsMonitor.js index 862daf4b59d..1d313c389f2 100644 --- a/ui/modules/connections/connectionsMonitor.js +++ b/ui/modules/connections/connectionsMonitor.js @@ -69,6 +69,7 @@ function onResetConnectionMetricsClick() { function onConnectionLogTableClick(event) { connectionLogDetail.setValue(JSON.stringify(connectionLogs[event.target.parentNode.rowIndex - 1], null, 2), -1); + connectionLogDetail.session.getUndoManager().reset(); } function onResetConnectionLogsClick() { @@ -104,6 +105,7 @@ function retrieveConnectionStatus() { Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); API.callConnectionsAPI('retrieveStatus', (connectionStatus) => { connectionStatusDetail.setValue(JSON.stringify(connectionStatus, null, 2), -1); + connectionStatusDetail.session.getUndoManager().reset(); }, selectedConnectionId); } diff --git a/ui/modules/things/features.js b/ui/modules/things/features.js index 5d5d813965d..b81c845c1c0 100644 --- a/ui/modules/things/features.js +++ b/ui/modules/things/features.js @@ -180,6 +180,8 @@ function updateFeatureEditors(featureJson) { featurePropertiesEditor.setValue(''); featureDesiredPropertiesEditor.setValue(''); } + featurePropertiesEditor.session.getUndoManager().reset(); + featureDesiredPropertiesEditor.session.getUndoManager().reset(); } /** diff --git a/ui/modules/things/messagesIncoming.js b/ui/modules/things/messagesIncoming.js index f9b0341aba9..86dbfd0c0cd 100644 --- a/ui/modules/things/messagesIncoming.js +++ b/ui/modules/things/messagesIncoming.js @@ -43,6 +43,7 @@ export function ready() { function onMessageTableClick(event) { messageDetail.setValue(JSON.stringify(messages[event.target.parentNode.rowIndex - 1], null, 2), -1); + messageDetail.session.getUndoManager().reset(); } function onResetMessagesClick() { diff --git a/ui/modules/things/thingsCRUD.js b/ui/modules/things/thingsCRUD.js index 93319eed8b8..4be416be9d9 100644 --- a/ui/modules/things/thingsCRUD.js +++ b/ui/modules/things/thingsCRUD.js @@ -154,10 +154,12 @@ function onThingChanged(thingJson) { delete thingCopy['_created']; delete thingCopy['_modified']; thingJsonEditor.setValue(JSON.stringify(thingCopy, null, 2), -1); + thingJsonEditor.session.getUndoManager().reset(); } else { dom.crudThings.idValue = null; dom.inputThingDefinition.value = null; thingJsonEditor.setValue(''); + thingJsonEditor.session.getUndoManager().reset(); } } } diff --git a/ui/modules/things/thingsSSE.js b/ui/modules/things/thingsSSE.js index ed46ac5c6c0..cafc3a99eb2 100644 --- a/ui/modules/things/thingsSSE.js +++ b/ui/modules/things/thingsSSE.js @@ -17,15 +17,18 @@ import * as API from '../api.js'; import * as Environments from '../environments/environments.js'; import * as Things from './things.js'; +import * as ThingsSearch from './thingsSearch.js'; -let thingEventSource; +let selectedThingEventSource; +let thingsTableEventSource; /** * Initializes components. Should be called after DOMContentLoaded event */ export async function ready() { - Things.addChangeListener(onThingChanged); + Things.addChangeListener(onSelectedThingChanged); + ThingsSearch.addChangeListener(onThingsTableChanged); Environments.addChangeListener(onEnvironmentChanged); } @@ -37,36 +40,52 @@ function notifyAll(thingJson) { observers.forEach(observer => observer.call(null, thingJson)); } -function onThingChanged(newThingJson, isNewThingId) { +function onThingsTableChanged(thingIds, fieldsQueryParameter) { + stopSSE(thingsTableEventSource); + if (thingIds) { + console.log('SSE Start: THINGS TABLE'); + thingsTableEventSource = API.getEventSource(thingIds, fieldsQueryParameter); + thingsTableEventSource.onmessage = onMessageThingsTable; + } +} + +function onSelectedThingChanged(newThingJson, isNewThingId) { if (!newThingJson) { - stopSSE(); + stopSSE(selectedThingEventSource); } else if (isNewThingId) { - thingEventSource && thingEventSource.close(); - console.log('Start SSE: ' + newThingJson.thingId); - thingEventSource = API.getEventSource(newThingJson.thingId, - '&fields=thingId,attributes,features,_revision,_modified'); - thingEventSource.onmessage = onMessage; + selectedThingEventSource && selectedThingEventSource.close(); + console.log('SSE Start: SELECTED THING : ' + newThingJson.thingId); + selectedThingEventSource = API.getEventSource(newThingJson.thingId, + 'fields=thingId,attributes,features,_revision,_modified'); + selectedThingEventSource.onmessage = onMessageSelectedThing; } } -function stopSSE() { - thingEventSource && thingEventSource.close(); - thingEventSource = null; - console.log('SSE Stopped'); +function stopSSE(eventSource) { + if (eventSource) { + eventSource.close(); + console.log('SSE Stopped: ' + (eventSource === selectedThingEventSource ? 'SELECTED THING' : 'THINGS TABLE')); + } } function onEnvironmentChanged(modifiedField) { if (!['pinnedThings', 'filterList', 'messageTemplates'].includes(modifiedField)) { - stopSSE(); + stopSSE(selectedThingEventSource); + stopSSE(thingsTableEventSource); } } -function onMessage(event) { +function onMessageSelectedThing(event) { if (event.data && event.data !== '') { - console.log(event); const merged = _.merge(Things.theThing, JSON.parse(event.data)); Things.setTheThing(merged); notifyAll(JSON.parse(event.data)); } } +function onMessageThingsTable(event) { + if (event.data && event.data !== '') { + ThingsSearch.updateTableRow(JSON.parse(event.data)); + } +} + diff --git a/ui/modules/things/thingsSearch.js b/ui/modules/things/thingsSearch.js index 7ca6f0b8787..5b9cf1b6c4d 100644 --- a/ui/modules/things/thingsSearch.js +++ b/ui/modules/things/thingsSearch.js @@ -14,6 +14,7 @@ /* eslint-disable require-jsdoc */ /* eslint-disable new-cap */ /* eslint-disable no-invalid-this */ +/* eslint-disable arrow-parens */ // @ts-check @@ -38,9 +39,18 @@ const dom = { favIcon: null, }; +const observers = []; +export function addChangeListener(observer) { + observers.push(observer); +} +function notifyAll(thingIds, fields) { + observers.forEach(observer => observer.call(null, thingIds, fields)); +} + + export async function ready() { Things.addChangeListener(onThingChanged); - ThingsSSE.addChangeListener(onSelectedThingUpdate); + ThingsSSE.addChangeListener(updateTableRow); Utils.getAllElementsById(dom); @@ -107,20 +117,27 @@ export function performLastSearch() { */ export function getThings(thingIds) { dom.thingsTableBody.innerHTML = ''; + const fieldsQueryParameter = Fields.getQueryParameter(); if (thingIds.length > 0) { API.callDittoREST('GET', - `/things?${Fields.getQueryParameter()}&ids=${thingIds}&option=sort(%2BthingId)`) - .then(fillThingsTable) + `/things?${fieldsQueryParameter}&ids=${thingIds}&option=sort(%2BthingId)`) + .then((thingJsonArray) => { + fillThingsTable(thingJsonArray); + notifyAll(thingIds, fieldsQueryParameter); + }) .catch((error) => { resetAndClearViews(); + notifyAll(null); }); } else { resetAndClearViews(); + notifyAll(null); } } function resetAndClearViews(retainThing = false) { theSearchCursor = null; + dom.thingsTableHead.innerHTML = ''; dom.thingsTableBody.innerHTML = ''; if (!retainThing) { Things.setTheThing(null); @@ -136,9 +153,9 @@ function searchThings(filter, isMore = false) { document.body.style.cursor = 'progress'; const namespaces = Environments.current().searchNamespaces; - + const fieldsQueryParameter = Fields.getQueryParameter(); API.callDittoREST('GET', - '/search/things?' + Fields.getQueryParameter() + + '/search/things?' + fieldsQueryParameter + ((filter && filter !== '') ? '&filter=' + encodeURIComponent(filter) : '') + ((namespaces && namespaces !== '') ? '&namespaces=' + namespaces : '') + '&option=sort(%2BthingId)' + @@ -152,8 +169,10 @@ function searchThings(filter, isMore = false) { } fillThingsTable(searchResult.items); checkMorePages(searchResult); + notifyAll(searchResult.items.map(thingJson => thingJson.thingId), fieldsQueryParameter); }).catch((error) => { resetAndClearViews(); + notifyAll(null); }).finally(() => { document.body.style.cursor = 'default'; }); @@ -273,7 +292,7 @@ function onThingChanged(thingJson) { } } -function onSelectedThingUpdate(thingUpdateJson) { +export function updateTableRow(thingUpdateJson) { const row = document.getElementById(thingUpdateJson.thingId); Array.from(row.cells).forEach((cell) => { const path = cell.getAttribute('jsonPath'); From 6cdc07e5b53ff63e03473f7a8586437f93039baa Mon Sep 17 00:00:00 2001 From: thfries Date: Sun, 22 Jan 2023 16:25:13 +0100 Subject: [PATCH 020/173] UI - SSE support - testing and bugfixing - WoT description for feature was referencing old dom field - thing search more button changed SSE to new page and lost 1st page Signed-off-by: thfries --- ui/modules/things/thingsSearch.js | 5 ++++- ui/modules/things/wotDescription.js | 15 ++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ui/modules/things/thingsSearch.js b/ui/modules/things/thingsSearch.js index 5b9cf1b6c4d..0acc59f0eab 100644 --- a/ui/modules/things/thingsSearch.js +++ b/ui/modules/things/thingsSearch.js @@ -169,7 +169,9 @@ function searchThings(filter, isMore = false) { } fillThingsTable(searchResult.items); checkMorePages(searchResult); - notifyAll(searchResult.items.map(thingJson => thingJson.thingId), fieldsQueryParameter); + if (!isMore) { + notifyAll(searchResult.items.map(thingJson => thingJson.thingId), fieldsQueryParameter); + } }).catch((error) => { resetAndClearViews(); notifyAll(null); @@ -294,6 +296,7 @@ function onThingChanged(thingJson) { export function updateTableRow(thingUpdateJson) { const row = document.getElementById(thingUpdateJson.thingId); + console.assert(row !== null, 'Unexpected thingId for table update. thingId was not loaded before'); Array.from(row.cells).forEach((cell) => { const path = cell.getAttribute('jsonPath'); if (path) { diff --git a/ui/modules/things/wotDescription.js b/ui/modules/things/wotDescription.js index 77032d652b0..f483c68dda9 100644 --- a/ui/modules/things/wotDescription.js +++ b/ui/modules/things/wotDescription.js @@ -21,7 +21,7 @@ export function WoTDescription(targetTab, forFeature) { let aceWoTDescription; let viewDirty = false; const _forFeature = forFeature; - const domTheFeatureId = document.getElementById('theFeatureId'); + let theFeatureId; const ready = async () => { const tabId = Utils.addTab( @@ -43,7 +43,10 @@ export function WoTDescription(targetTab, forFeature) { aceWoTDescription = Utils.createAceEditor(aceId, 'ace/mode/json', true); }; - const onReferenceChanged = () => { + const onReferenceChanged = (ref) => { + if (_forFeature) { + theFeatureId = ref; + } if (tabLink && tabLink.classList.contains('active')) { refreshDescription(); } else { @@ -64,12 +67,10 @@ export function WoTDescription(targetTab, forFeature) { } function refreshDescription() { - let featurePath = ''; - if (_forFeature) { - featurePath = '/features/' + domTheFeatureId.value; - } + const featurePath = _forFeature ? '/features/' + theFeatureId : ''; + aceWoTDescription.setValue(''); - if (Things.theThing && (!_forFeature || domTheFeatureId.value)) { + if (Things.theThing && (!_forFeature || theFeatureId)) { API.callDittoREST( 'GET', '/things/' + Things.theThing.thingId + featurePath, From 948b85dc2049713ae28d382097732e57749f836d Mon Sep 17 00:00:00 2001 From: Andrey Balarev Date: Mon, 23 Jan 2023 21:51:28 +0200 Subject: [PATCH 021/173] Connections API doc fixes Ditto Signed-off-by: Andrey Balarev --- .../service/messaging/hono/HonoConnectionFactory.java | 5 ++++- .../messaging/hono/DefaultHonoConnectionFactoryTest.java | 6 ++++-- .../src/test/resources/hono-connection-custom-expected.json | 2 +- connectivity/service/src/test/resources/test.conf | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/HonoConnectionFactory.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/HonoConnectionFactory.java index 3057a0ea4d9..8552aee4b5a 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/HonoConnectionFactory.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/HonoConnectionFactory.java @@ -16,6 +16,8 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.util.Collection; import java.util.LinkedHashMap; @@ -150,7 +152,8 @@ protected void preConversion(final Connection honoConnection) { private String combineUriWithCredentials(final String uri, final UserPasswordCredentials credentials) { return uri.replaceFirst("(\\S+://)(\\S+)", - "$1" + credentials.getUsername() + ":" + credentials.getPassword() + "@$2"); + "$1" + URLEncoder.encode(credentials.getUsername(), StandardCharsets.UTF_8) + ":" + + URLEncoder.encode(credentials.getPassword(), StandardCharsets.UTF_8) + "@$2"); } private Map makeupSpecificConfig(final Connection connection) { diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java index 6a72dd50171..86a6bd2443e 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java @@ -19,6 +19,8 @@ import java.io.IOException; import java.io.InputStreamReader; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -120,8 +122,8 @@ private Connection getExpectedHonoConnection(final Connection originalConnection ); return ConnectivityModelFactory.newConnectionBuilder(originalConnection) .uri(honoConfig.getBaseUri().toString().replaceFirst("(\\S+://)(\\S+)", - "$1" + honoConfig.getUserPasswordCredentials().getUsername() - + ":" + honoConfig.getUserPasswordCredentials().getPassword() + "$1" + URLEncoder.encode(honoConfig.getUserPasswordCredentials().getUsername(), StandardCharsets.UTF_8) + + ":" + URLEncoder.encode(honoConfig.getUserPasswordCredentials().getPassword(), StandardCharsets.UTF_8) + "@$2")) .validateCertificate(honoConfig.isValidateCertificates()) .specificConfig(Map.of( diff --git a/connectivity/service/src/test/resources/hono-connection-custom-expected.json b/connectivity/service/src/test/resources/hono-connection-custom-expected.json index c55d5c52168..649861792bb 100644 --- a/connectivity/service/src/test/resources/hono-connection-custom-expected.json +++ b/connectivity/service/src/test/resources/hono-connection-custom-expected.json @@ -3,7 +3,7 @@ "name": "Things-Hono Test 1", "connectionType": "hono", "connectionStatus": "open", - "uri": "tcp://test_username:test_password@localhost:9922", + "uri": "tcp://test_username:test_password_w%2Fspecial_char@localhost:9922", "sources": [ { "addresses": [ diff --git a/connectivity/service/src/test/resources/test.conf b/connectivity/service/src/test/resources/test.conf index b4ec24e9b1c..a4676ce04ff 100644 --- a/connectivity/service/src/test/resources/test.conf +++ b/connectivity/service/src/test/resources/test.conf @@ -103,7 +103,7 @@ ditto { sasl-mechanism = PLAIN bootstrap-servers = "tcp://server1:port1,tcp://server2:port2,tcp://server3:port3" username = test_username - password = test_password + password = test_password_w/special_char } connection { From e76bc2d48fc64c7fca9d4cd8454c69766545b928 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Wed, 25 Jan 2023 10:09:42 +0100 Subject: [PATCH 022/173] fixed Grafana dashboard datasource configurations Signed-off-by: Thomas Jaeckle --- .../operations/grafana-dashboards/Akka.json | 20 +-- .../Akka_Dispatcher_Metrics.json | 115 +++---------- .../grafana-dashboards/Cache_Metrics.json | 2 +- .../grafana-dashboards/Cluster_traffic.json | 37 +--- .../grafana-dashboards/Connectivity_ACKS.json | 2 +- .../Connectivity_Metrics.json | 160 ++++-------------- .../Connectivity_live_status.json | 2 +- .../grafana-dashboards/Container_Metrics.json | 2 +- .../grafana-dashboards/External_Metrics.json | 2 +- .../grafana-dashboards/Gateway_Traces.json | 2 +- .../grafana-dashboards/JVM_Metrics.json | 2 +- .../Kafka_Consumer_Metrics.json | 2 +- .../Kubernetes_Metrics.json | 2 +- .../grafana-dashboards/Load_Test.json | 50 ++---- .../Persistence_Entities.json | 2 +- .../grafana-dashboards/Pub_Sub.json | 50 ++---- .../grafana-dashboards/Signal_processing.json | 140 +++------------ .../Sudo_command_count.json | 45 +---- ...s-Wildcard-Search_Performance_Metrics.json | 75 ++------ pom.xml | 6 +- 20 files changed, 152 insertions(+), 566 deletions(-) diff --git a/deployment/operations/grafana-dashboards/Akka.json b/deployment/operations/grafana-dashboards/Akka.json index 75d5b19244b..32d6e089eda 100644 --- a/deployment/operations/grafana-dashboards/Akka.json +++ b/deployment/operations/grafana-dashboards/Akka.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -29,10 +26,7 @@ "liveNow": false, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -130,10 +124,7 @@ "text": "None", "value": "" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(actor_mailbox_size, service)", "hide": 0, "includeAll": false, @@ -159,10 +150,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(actor_mailbox_size{service=~\"$service\"}, instance)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Akka_Dispatcher_Metrics.json b/deployment/operations/grafana-dashboards/Akka_Dispatcher_Metrics.json index f2dff2f4b2e..9b36113fb79 100644 --- a/deployment/operations/grafana-dashboards/Akka_Dispatcher_Metrics.json +++ b/deployment/operations/grafana-dashboards/Akka_Dispatcher_Metrics.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -34,10 +31,7 @@ { "collapse": false, "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -52,10 +46,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Tracks the number of threads in use.", "fieldConfig": { "defaults": { @@ -131,10 +122,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_threads_total_sum{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])\n/\nrate(executor_threads_total_count{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -144,10 +132,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_threads_active_sum{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])\n/\nrate(executor_threads_active_count{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -166,10 +151,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -211,10 +193,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_tasks_submitted_total{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -225,10 +204,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_tasks_completed_total{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -239,10 +215,7 @@ "refId": "B" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "max_over_time( ( rate( executor_tasks_completed_total{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval] ) )[$__interval:30s] )", "format": "time_series", @@ -284,10 +257,7 @@ } }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -366,10 +336,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "rate(executor_queue_size_sum{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])\n/\nrate(executor_queue_size_count{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", "hide": false, @@ -378,10 +345,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(executor_queue_size_bucket{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\",le=\"+Inf\"}[$Interval])) by (job,instance,name,le))", "format": "time_series", @@ -396,10 +360,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Tracks minimum/maximum number of Threads of the executors.", "fieldConfig": { "defaults": { @@ -479,10 +440,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "executor_threads_min{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}", "format": "time_series", @@ -492,10 +450,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "executor_threads_max{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}", "format": "time_series", @@ -514,10 +469,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Tracks executor parallelism.", "fill": 1, "fillGradient": 0, @@ -560,10 +512,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "executor_parallelism{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}", "format": "time_series", @@ -604,10 +553,7 @@ } }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Tracks the time that tasks spend on the executor service's queue", "fieldConfig": { "defaults": { @@ -687,10 +633,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "rate(executor_time_in_queue_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])\n/\nrate(executor_time_in_queue_seconds_count{job=~\"$Application\",instance=~\"$Instance\",name=~\"$Dispatcher\",type=~\"$Type\"}[$Interval])", "format": "time_series", @@ -723,10 +666,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values({__name__=~\"jvm.*\"}, job)", "hide": 0, "includeAll": true, @@ -757,10 +697,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(executor_threads_min{job=~\"$Application\"},instance)", "hide": 0, "includeAll": true, @@ -791,10 +728,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(executor_threads_min{job=~\"$Application\",instance=~\"$Instance\"},name)", "hide": 0, "includeAll": true, @@ -880,10 +814,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(executor_threads_min{job=~\"$Application\",instance=~\"$Instance\"},type)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Cache_Metrics.json b/deployment/operations/grafana-dashboards/Cache_Metrics.json index 4cae8324023..5eb25da45df 100644 --- a/deployment/operations/grafana-dashboards/Cache_Metrics.json +++ b/deployment/operations/grafana-dashboards/Cache_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Cluster_traffic.json b/deployment/operations/grafana-dashboards/Cluster_traffic.json index a837a2ad0b1..8723f3e16b0 100644 --- a/deployment/operations/grafana-dashboards/Cluster_traffic.json +++ b/deployment/operations/grafana-dashboards/Cluster_traffic.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -680,10 +680,7 @@ }, { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -696,10 +693,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -779,10 +773,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(enforcement_seconds_count{segment=\"overall\",channel!=\"live\"}[5m]) > 0) by (category, resource, outcome)", "format": "time_series", @@ -792,10 +783,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(enforcement_seconds_count{segment=\"overall\",channel=\"live\"}[5m]) > 0) by (category, resource, outcome)", "format": "time_series", @@ -810,10 +798,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -890,10 +875,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "avg((idelta(enforcement_seconds_sum{segment=\"overall\",channel!=\"live\"}[5m]) / idelta(enforcement_seconds_count{segment=\"overall\",channel!=\"live\"}[5m])) > 0) by (category, resource, outcome)", "format": "time_series", @@ -903,10 +885,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "avg((idelta(enforcement_seconds_sum{segment=\"overall\",channel=\"live\"}[5m]) / idelta(enforcement_seconds_count{segment=\"overall\",channel=\"live\"}[5m])) > 0) by (category, resource, outcome)", "format": "time_series", diff --git a/deployment/operations/grafana-dashboards/Connectivity_ACKS.json b/deployment/operations/grafana-dashboards/Connectivity_ACKS.json index b47c75f32a2..8ac2860b141 100644 --- a/deployment/operations/grafana-dashboards/Connectivity_ACKS.json +++ b/deployment/operations/grafana-dashboards/Connectivity_ACKS.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Connectivity_Metrics.json b/deployment/operations/grafana-dashboards/Connectivity_Metrics.json index 33822617a26..2a769a70426 100644 --- a/deployment/operations/grafana-dashboards/Connectivity_Metrics.json +++ b/deployment/operations/grafana-dashboards/Connectivity_Metrics.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -30,10 +27,7 @@ "panels": [ { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -50,10 +44,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -102,10 +93,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "sum(connection_client{instance=~\"$Host\", type=~\"$ConnectionType\", id=~\"$Connection\"}) by (type)", "format": "time_series", "instant": false, @@ -149,10 +137,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -201,10 +186,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "sum(connection_client{instance=~\"$Host\", type=~\"$ConnectionType\", id=~\"$Connection\"}) by (instance, type)", "format": "time_series", "instant": false, @@ -248,10 +230,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Connections which delivered metrics in the last 7 days", "fieldConfig": { "defaults": { @@ -301,10 +280,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "count by (ditto_connection_type) (count by (instance, ditto_connection_id, ditto_connection_type) (increase(connectivity_message_mapping_seconds_count{instance=~\"$Host\", ditto_connection_type=~\"$ConnectionType\", ditto_connection_id=~\"$Connection\"}[5m]) > 0))", "format": "time_series", "hide": false, @@ -349,10 +325,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Connections which delivered metrics in the last 7 days", "fieldConfig": { "defaults": { @@ -402,10 +375,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "expr": "count by (instance, ditto_connection_type) (count by (instance, ditto_connection_id, ditto_connection_type) (increase (connectivity_message_mapping_seconds_count{instance=~\"$Host\", ditto_connection_type=~\"$ConnectionType\", ditto_connection_id=~\"$Connection\"}[5m]) > 0))", "format": "time_series", "hide": false, @@ -447,10 +417,7 @@ }, { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -467,10 +434,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -566,10 +530,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "This number reflects the number of messages that resulted out of payload mappings per second.", "fieldConfig": { "defaults": { @@ -666,10 +627,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -766,10 +724,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -871,10 +826,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1006,10 +958,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1056,10 +1005,7 @@ "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": true, "expr": "count(rate(connection_messages_total{instance=~\"$Host\", id=~\"$Connection\", type=~\"$ConnectionType\", category=\"throttled\", direction=\"inbound\"}[5m])>0) by (type, id)", @@ -1071,10 +1017,7 @@ "refId": "B" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "exemplar": true, "expr": "count(rate(connection_messages_total{category=\"throttled\", direction=\"inbound\"}[5m]) > 0) by (id)", "hide": true, @@ -1131,10 +1074,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1228,10 +1168,7 @@ }, { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -1248,10 +1185,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1348,10 +1282,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1447,10 +1378,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "This number reflects the number of messages that resulted out of payload mappings per second.", "fieldConfig": { "defaults": { @@ -1547,10 +1475,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1646,10 +1571,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1745,10 +1667,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [], @@ -1844,10 +1763,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -1962,10 +1878,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -2074,10 +1987,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "", "hide": 0, "includeAll": true, @@ -2108,10 +2018,7 @@ "$__all" ] }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(connectivity_message_mapping_seconds_count,ditto_connection_id)", "hide": 0, "includeAll": true, @@ -2138,10 +2045,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(connectivity_message_mapping_seconds_count,ditto_connection_type)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Connectivity_live_status.json b/deployment/operations/grafana-dashboards/Connectivity_live_status.json index 52796004638..3b3f266c3d9 100644 --- a/deployment/operations/grafana-dashboards/Connectivity_live_status.json +++ b/deployment/operations/grafana-dashboards/Connectivity_live_status.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Container_Metrics.json b/deployment/operations/grafana-dashboards/Container_Metrics.json index a3949c79f01..832cd2d6a82 100644 --- a/deployment/operations/grafana-dashboards/Container_Metrics.json +++ b/deployment/operations/grafana-dashboards/Container_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/External_Metrics.json b/deployment/operations/grafana-dashboards/External_Metrics.json index a21553f3643..67b79c3f72d 100644 --- a/deployment/operations/grafana-dashboards/External_Metrics.json +++ b/deployment/operations/grafana-dashboards/External_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Gateway_Traces.json b/deployment/operations/grafana-dashboards/Gateway_Traces.json index 3de7a4cb297..a9350acf75d 100644 --- a/deployment/operations/grafana-dashboards/Gateway_Traces.json +++ b/deployment/operations/grafana-dashboards/Gateway_Traces.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/JVM_Metrics.json b/deployment/operations/grafana-dashboards/JVM_Metrics.json index 2c7fdfe5920..21a526f700e 100644 --- a/deployment/operations/grafana-dashboards/JVM_Metrics.json +++ b/deployment/operations/grafana-dashboards/JVM_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Kafka_Consumer_Metrics.json b/deployment/operations/grafana-dashboards/Kafka_Consumer_Metrics.json index ef6a63d89da..2d468fbf34d 100644 --- a/deployment/operations/grafana-dashboards/Kafka_Consumer_Metrics.json +++ b/deployment/operations/grafana-dashboards/Kafka_Consumer_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Kubernetes_Metrics.json b/deployment/operations/grafana-dashboards/Kubernetes_Metrics.json index 72c17b5d1af..ddecf3ff22d 100644 --- a/deployment/operations/grafana-dashboards/Kubernetes_Metrics.json +++ b/deployment/operations/grafana-dashboards/Kubernetes_Metrics.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Load_Test.json b/deployment/operations/grafana-dashboards/Load_Test.json index 514a65c2095..6c0544e7301 100644 --- a/deployment/operations/grafana-dashboards/Load_Test.json +++ b/deployment/operations/grafana-dashboards/Load_Test.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -29,10 +26,7 @@ "panels": [ { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -45,10 +39,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -132,10 +123,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -210,10 +198,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(enforcement_seconds_count{segment=\"overall\",channel!=\"live\"}[5m]) > 0) by (category, resource, outcome)", "format": "time_series", @@ -223,10 +208,7 @@ "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(enforcement_seconds_count{segment=\"overall\",channel=\"live\"}[5m]) > 0) by (category, resource, outcome)", "format": "time_series", @@ -242,10 +224,7 @@ }, { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "gridPos": { "h": 1, "w": 24, @@ -258,10 +237,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -354,10 +330,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -444,10 +417,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Positive: inbound messages\nNegative: outbound messages", "fieldConfig": { "defaults": { diff --git a/deployment/operations/grafana-dashboards/Persistence_Entities.json b/deployment/operations/grafana-dashboards/Persistence_Entities.json index a89f0319c5f..f69715730da 100644 --- a/deployment/operations/grafana-dashboards/Persistence_Entities.json +++ b/deployment/operations/grafana-dashboards/Persistence_Entities.json @@ -3,7 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/deployment/operations/grafana-dashboards/Pub_Sub.json b/deployment/operations/grafana-dashboards/Pub_Sub.json index 49135cb536c..a88ae765a2e 100644 --- a/deployment/operations/grafana-dashboards/Pub_Sub.json +++ b/deployment/operations/grafana-dashboards/Pub_Sub.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -33,10 +30,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -120,10 +114,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -222,10 +213,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -305,10 +293,7 @@ } }, { - "datasource": { - "type": "elasticsearch", - "uid": "PAE1B8C8635429669" - }, + "datasource": "elasticsearch", "fieldConfig": { "defaults": { "color": { @@ -555,10 +540,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -645,10 +627,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fill": 1, "fillGradient": 0, @@ -770,10 +749,7 @@ "noDataState": "no_data", "notifications": [] }, - "datasource": { - "type": "elasticsearch", - "uid": "PAE1B8C8635429669" - }, + "datasource": "elasticsearch", "fieldConfig": { "defaults": { "color": { @@ -882,10 +858,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -981,10 +954,7 @@ "templating": { "list": [ { - "datasource": { - "type": "elasticsearch", - "uid": "PAE1B8C8635429669" - }, + "datasource": "elasticsearch", "filters": [], "hide": 0, "name": "Filters", diff --git a/deployment/operations/grafana-dashboards/Signal_processing.json b/deployment/operations/grafana-dashboards/Signal_processing.json index 3a4e8f9ae20..d02ef2c1fe2 100644 --- a/deployment/operations/grafana-dashboards/Signal_processing.json +++ b/deployment/operations/grafana-dashboards/Signal_processing.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -42,10 +39,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -135,10 +129,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "sum(irate(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"overall\",outcome=~\"$Outcome\",channel!=\"live\"}[$__rate_interval])) by (resource, category, outcome)", @@ -152,10 +143,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -248,10 +236,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "sum(irate(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"overall\",channel=\"live\"}[$__rate_interval])) by (resource, category, outcome)", @@ -266,10 +251,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -341,10 +323,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"overall\",channel!=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"overall\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -358,10 +337,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -433,10 +409,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"overall\",channel=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"overall\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -464,10 +437,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -539,10 +509,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(pre_enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",channel!=\"live\"}[$__rate_interval]) / idelta(pre_enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -556,10 +523,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -631,10 +595,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(pre_enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",channel=\"live\"}[$__rate_interval]) / idelta(pre_enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -662,10 +623,7 @@ "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -737,10 +695,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"enf\",channel!=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"enf\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -754,10 +709,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -829,10 +781,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",segment=\"enf\",channel=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"enf\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -857,10 +806,7 @@ "id": 17, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -971,10 +917,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"process\",channel!=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"process\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -988,10 +931,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1063,10 +1003,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"process\",channel=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"process\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -1095,10 +1032,7 @@ "id": 21, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1170,10 +1104,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"resp_filter\",channel!=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"resp_filter\",channel!=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -1187,10 +1118,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "", "fieldConfig": { "defaults": { @@ -1262,10 +1190,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "avg((idelta(enforcement_seconds_sum{job=~\"$Application\",instance=~\"$Instance\",outcome=~\"$Outcome\",segment=\"resp_filter\",channel=\"live\"}[$__rate_interval]) / idelta(enforcement_seconds_count{job=~\"$Application\",instance=~\"$Instance\",segment=\"resp_filter\",channel=\"live\"}[$__rate_interval])) > 0) by (resource, category, outcome)", @@ -1297,10 +1222,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(enforcement_seconds_count, job)", "hide": 0, "includeAll": true, @@ -1324,10 +1246,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(enforcement_seconds_count{job=\"$Application\"}, instance)", "hide": 0, "includeAll": true, @@ -1351,10 +1270,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(enforcement_seconds_count, outcome)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Sudo_command_count.json b/deployment/operations/grafana-dashboards/Sudo_command_count.json index a25d5992242..587476ea225 100644 --- a/deployment/operations/grafana-dashboards/Sudo_command_count.json +++ b/deployment/operations/grafana-dashboards/Sudo_command_count.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -29,10 +26,7 @@ "liveNow": false, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -105,10 +99,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "expr": "sum(irate(sudo_commands_total{job=~\"$Application\",instance=~\"$Instance\"}[5m])) by (type)", "legendFormat": "{{type}}", @@ -120,10 +111,7 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -189,10 +177,7 @@ "pluginVersion": "8.5.6", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "sum(sudo_commands_total{job=~\"$Application\",instance=~\"$Instance\"}) by (type)", @@ -241,10 +226,7 @@ "type": "table" }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -290,10 +272,7 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editorMode": "code", "exemplar": false, "expr": "sum(sudo_commands_total{job=~\"$Application\",instance=~\"$Instance\"}) by (type)", @@ -334,10 +313,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(sudo_commands_total, job)", "hide": 0, "includeAll": true, @@ -361,10 +337,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "label_values(sudo_commands_total, instance)", "hide": 0, "includeAll": true, diff --git a/deployment/operations/grafana-dashboards/Things-Wildcard-Search_Performance_Metrics.json b/deployment/operations/grafana-dashboards/Things-Wildcard-Search_Performance_Metrics.json index 29353fd8d7a..6637c5084a0 100644 --- a/deployment/operations/grafana-dashboards/Things-Wildcard-Search_Performance_Metrics.json +++ b/deployment/operations/grafana-dashboards/Things-Wildcard-Search_Performance_Metrics.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "grafana", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -35,10 +32,7 @@ "cacheTimeout": "0", "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Positive Y axis: stacked search queries\nNegative Y axis: stacked search counts", "editable": true, "error": false, @@ -143,10 +137,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editable": true, "error": false, "fieldConfig": { @@ -242,10 +233,7 @@ "cacheTimeout": "0", "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "editable": true, "error": false, "fieldConfig": { @@ -343,10 +331,7 @@ "cacheTimeout": "0", "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "description": "Percentiles are capped to max. 10s", "editable": true, "error": false, @@ -475,10 +460,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -568,10 +550,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -675,10 +654,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fill": 1, "fillGradient": 0, "gridPos": { @@ -764,10 +740,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -857,10 +830,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -990,10 +960,7 @@ } }, { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "color": { @@ -1087,10 +1054,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -1180,10 +1144,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -1273,10 +1234,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "fieldConfig": { "defaults": { "links": [] @@ -1383,10 +1341,7 @@ "text": "All", "value": "$__all" }, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, + "datasource": "prometheus", "definition": "", "hide": 0, "includeAll": true, diff --git a/pom.xml b/pom.xml index 03e36f5c9cb..b8c9190c220 100644 --- a/pom.xml +++ b/pom.xml @@ -233,9 +233,9 @@ yyyy-MM-dd ${maven.build.timestamp} - scm:git:git@github.com:eclipse/ditto.git - scm:git:https://github.com/eclipse/ditto.git - https://github.com/eclipse/ditto.git + scm:git:git@github.com:eclipse-ditto/ditto.git + scm:git:https://github.com/eclipse-ditto/ditto.git + https://github.com/eclipse-ditto/ditto.git -Dfile.encoding=${project.build.sourceEncoding} From d8f97cdba746fb4dc9f33e785a3f83345c66be0f Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Wed, 25 Jan 2023 17:22:05 +0100 Subject: [PATCH 023/173] added example prometheus.yml config for a Ditto deployment Signed-off-by: Thomas Jaeckle --- .../operations/prometheus/prometheus.yml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 deployment/operations/prometheus/prometheus.yml diff --git a/deployment/operations/prometheus/prometheus.yml b/deployment/operations/prometheus/prometheus.yml new file mode 100644 index 00000000000..a5a50a5676f --- /dev/null +++ b/deployment/operations/prometheus/prometheus.yml @@ -0,0 +1,42 @@ +global: + scrape_interval: 30s + scrape_timeout: 10s + evaluation_interval: 30s + +scrape_configs: + # Scrape prometheus itself. + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # Scrape grafana. + - job_name: 'grafana' + static_configs: + - targets: ['grafana:3000'] + + # Scrape Ditto. + - job_name: 'ditto' + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_path ] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: ${1}:${2} + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: instance + - source_labels: [__meta_kubernetes_pod_container_name] + action: replace + target_label: job From 987f94ecde1745b767218d65a838db60baba0fc0 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Thu, 26 Jan 2023 14:44:13 +0100 Subject: [PATCH 024/173] added 30 minutes overview of Eclipse Ditto slides Signed-off-by: Thomas Jaeckle --- .../resources/pages/ditto/presentations.md | 7 + .../slides/2023_01_ditto-in-30-min/index.html | 581 ++++++++++++++++++ 2 files changed, 588 insertions(+) create mode 100644 documentation/src/main/resources/slides/2023_01_ditto-in-30-min/index.html diff --git a/documentation/src/main/resources/pages/ditto/presentations.md b/documentation/src/main/resources/pages/ditto/presentations.md index 1b1b4ec6fb2..7d6b106c737 100644 --- a/documentation/src/main/resources/pages/ditto/presentations.md +++ b/documentation/src/main/resources/pages/ditto/presentations.md @@ -7,6 +7,13 @@ topnav: topnav This page contains a collection of presentations, videos and workshops about Eclipse Ditto, sorted from most recent ones downwards. +## 26.01.2023 Eclipse Ditto in 30 minutes + +Topic: Short 30 minutes overview of Eclipse Ditto + +The slides can be found here: [2023_01_ditto-in-30-min](slides/2023_01_ditto-in-30-min/index.html). + + ## 24.10.2022 EclipseCon Europe 2022 Community Day Topic: Project status update of Eclipse Ditto diff --git a/documentation/src/main/resources/slides/2023_01_ditto-in-30-min/index.html b/documentation/src/main/resources/slides/2023_01_ditto-in-30-min/index.html new file mode 100644 index 00000000000..825e4784080 --- /dev/null +++ b/documentation/src/main/resources/slides/2023_01_ditto-in-30-min/index.html @@ -0,0 +1,581 @@ + + + + + + + + Eclipse Ditto™: An introduction + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ +
+
+ +

+ Eclipse IoT + Eclipse Ditto +

+

Eclipse Ditto™
-
an introduction

+

01/2023

+
+
+ + +
+
+ +

Digital Twins

+
+
    +
  • digital representation of physical devices
  • +
  • twin as broker for communicating with assets
  • +
  • applicable for both industrial and consumer-centric IoT scenarios
  • +
+
+
+

Twins in scope of Ditto

+
+
    +
  • a pattern for working with things in the IoT
  • +
  • provide state persistence and search capabilities
  • +
  • access twins always in an authorized way
  • +
  • provide APIs - Device as a Service
  • +
  • optionally normalize device payloads
  • +
+
+
+ + +
+
+

Eclipse Ditto in context

+
+ Ditto in action +
+
+

Ditto as
Digital Twin
"middleware"

+
+
+
+

turn device data into APIs

+
+
{
+  "thingId": "io.foo:car1",
+  "policyId": "io.foo:car1",
+  "attributes": {
+    "manufacturer": "Foo",
+    "data": {
+      "serialNo": 4711
+    }
+  },
+  "features": {
+    "temp": {
+      "properties": {
+        "value": 23.42
+      }
+    }
+  }
+}
+

JSON repr. of a Thing

+
+
+
GET/PUT/DELETE /api/2/things/io.foo:car1
+ /api/2/things/io.foo:car1/thingId
+ /api/2/things/io.foo:car1/policyId
+ /api/2/things/io.foo:car1/attributes
+ /api/2/things/io.foo:car1/attributes/manufacturer
+ /api/2/things/io.foo:car1/attributes/data
+ /api/2/things/io.foo:car1/attributes/data/serialNo
+
+
+ /api/2/things/io.foo:car1/features
+ /api/2/things/io.foo:car1/features/temp
+ /api/2/things/io.foo:car1/features/temp/properties
+ /api/2/things/io.foo:car1/features/temp/properties/value
+
+
+
+
+
+

HTTP API of the Thing

+ → docs +
+
+
+

modeling thing capabilities

+
+
    +
  • by default, thing attributes and feature properties are "schemaless"
  • +
  • a thing may be aware of one "definition" +
  • +
  • a feature may be aware of several "definitions"
  • +
+
+
+
{
+  "thingId": "io.foo:lamp-1",
+  "policyId": "io.foo:lamp-1",
+  "definition": "https://some.domain/floor-lamp-1.0.0.tm.jsonld",
+  "attributes": {
+    "Manufacturer": "Foo corp",
+    "serialNo": "4711"
+  },
+  "features": {
+    "Spot1": {
+      "definition": [
+        "https://some.domain/dimmable-colored-lamp-1.0.0.tm.jsonld",
+        "https://some.domain/colored-lamp-1.0.0.tm.jsonld",
+        "https://some.domain/switchable-1.0.0.tm.jsonld"
+      ],
+      "properties": {
+        "on": true,
+        "color": {...
+        }
+      }
+    }
+  }
+}
+ → docs +
+
+
+

modeling thing capabilities

+ W3C Web of Things logo +
    +
  • integration of W3C Web of Things (WoT) standard
  • +
  • + reference in "definition" to WoT TMs (Thing Models) +
  • +
  • + let Ditto automatically generate WoT TDs (Thing Descriptions) containing API endpoints and data formats + + semantic annotations +
  • +
+
+
+

persistence of device state

+
+
    +
  • devices are not always connected to the net
  • +
  • applications always need to be able to access their data
  • +
  • twin vs. live access on API level
  • +
+
+
+ Ditto twin channel + Ditto live channel +
+
+
+

authorization

+
+
+
+
    +
  • Ditto contains a built-in authorization mechanism (Policies)
  • +
  • every API call is authorized
  • +
+
+
+
{
+    "policyId": "io.foo:car1-policy",
+    "entries": {
+      "owner": {
+        "subjects": {
+          "oidc:userid": {
+            "type": "OpenID connect extracted userid"
+          },
+          "oidc:/claim": {
+            "type": "OpenID connect extracted claim"
+          }
+        },
+        "resources": {
+          "thing:/": {
+            "grant": ["READ","WRITE"],
+            "revoke": []
+          },
+          "thing:/features/firmware": {
+            "grant": [],
+            "revoke": ["WRITE"]
+          },
+          "policy:/": {
+            "grant": ["READ","WRITE"],
+            "revoke": []
+          }
+        }
+      }
+    }
+  }
+ → docs +
+
+
+

search

+
+
+ Meme Dino +
    +
  • you must not
  • +
  • Ditto has you covered
  • +
+
+
+
GET /api/2/search/things
+  ?filter=like(attributes/manufacturer,"Foo*")
+
GET /api/2/search/things
+  ?filter=and(
+    eq(features/*/definition,"https://some.domain/switchable-1.0.0.tm.jsonld"),
+    like(attributes/manufacturer,"Foo*"),
+    not(gt(attributes/counter,42))
+  )
+  &fields=thingId,attributes/manufacturer,features/*/properties/on
+
+
+
    +
  • search for arbitrary data with RQL query
  • +
  • Ditto again ensures authorization
  • +
  • apply field projection over the results
  • +
  • don't worry about indexing
  • +
+ → docs +
+
+
+

get notified about changes

+
+
    +
  • notification via various channels: WebSocket, SSE, MQTT (3.1.1 | 5), AMQP (0.9.1 | 1.0), Apache Kafka, HTTP hook
  • +
  • server side filtering via RQL (same as in search)
  • +
+
+ → docs +
+
+
+

payload normalization

+
+
+
    +
  • devices send data in various formats
  • +
  • Ditto provides structured APIs of things (attributes, features)
  • +
  • devices don't need to be aware of Ditto
  • +
+
+
+
+ JavaScript logo + +
+
    +
  • incoming and outgoing data can be transformed
  • +
+
+
+
+

nonfunctional

+
+
+ Ditto context overview +
+
+
    +
  • modular architecture of Ditto services
  • +
  • horizontal scalability of each Ditto service
  • +
  • runtime dependency to MongoDB
  • +
  • included monitoring (JVM metrics, roundtrips, MongoDB)
  • +
  • codebase written in: Java
  • +
+
+
+
+ + +
+

Demo

+ + Ditto UI Screenshot + +
+ + +
+
+
+

Links

+
+ +
+
+
+ +
+ + + + + + + + + + + + + + + From 3f846a5b3df1d4a51452f982ad766a51c0722ef6 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Fri, 27 Jan 2023 09:05:05 +0100 Subject: [PATCH 025/173] improved documentation on token handling with e.g. "oauth2-proxy" Signed-off-by: Thomas Jaeckle --- .../main/resources/pages/ditto/installation-operating.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/documentation/src/main/resources/pages/ditto/installation-operating.md b/documentation/src/main/resources/pages/ditto/installation-operating.md index 9634b5aea47..aa90da3fbaf 100644 --- a/documentation/src/main/resources/pages/ditto/installation-operating.md +++ b/documentation/src/main/resources/pages/ditto/installation-operating.md @@ -157,9 +157,12 @@ The configured subject-issuer will be used to prefix the value of each individua As of the OAuth2.0 and OpenID Connect standards Ditto expects the headers `Authorization: Bearer ` and `Content-Type: application/json`, containing the issued token of the provider. -**The token has to be issued beforehand. The required logic is not provided by Ditto.** When using -the OIDC provider [keycloak](https://www.keycloak.org/), a project like [keycloak-gatekeeper](https://github.com/keycloak/keycloak-gatekeeper) -may be put in front of Ditto to handle the token-logic. +**The token has to be issued beforehand. The required logic is not provided by Ditto.** + +This can e.g. be done by an OIDC provider like [Keycloak](https://www.keycloak.org/). +A project like [oauth2-proxy](https://github.com/oauth2-proxy/oauth2-proxy) +may be put in front of Ditto to handle the token-logic like e.g. loading/saving the token from/to a Cookie and passing +it to Ditto as `Authorization` header. **If the chosen OIDC provider uses a self-signed certificate**, the certificate has to be retrieved and configured for the akka-http ssl configuration. From bedaaee80d093b1b577a84bce4511ce9bb096a17 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Fri, 27 Jan 2023 16:51:49 +0100 Subject: [PATCH 026/173] enhanced Connections with their "_revision", "_created" and "_modified" information * and made those queryable with "fields" selector same as for things+policies * made connection a full "Entity" Signed-off-by: Thomas Jaeckle --- connectivity/model/pom.xml | 2 +- .../model/AbstractConnection.java | 79 ++++++++++- .../model/AbstractConnectionBuilder.java | 22 +++ .../ditto/connectivity/model/Connection.java | 38 ++++-- .../connectivity/model/ConnectionBuilder.java | 39 ++++++ .../model/ConnectionRevision.java | 32 +++++ .../model/ConnectivityModelFactory.java | 11 ++ .../model/ImmutableConnectionRevision.java | 101 ++++++++++++++ .../signals/commands/ConnectivityCommand.java | 24 +++- ...ConnectionPreconditionFailedException.java | 126 ++++++++++++++++++ ...ctionPreconditionNotModifiedException.java | 125 +++++++++++++++++ .../commands/query/RetrieveConnection.java | 72 ++++++++-- .../commands/query/RetrieveConnections.java | 55 +++++++- .../query/RetrieveResolvedHonoConnection.java | 58 +++++++- .../query/RetrieveConnectionTest.java | 3 +- .../query/RetrieveConnectionsTest.java | 5 +- .../RetrieveResolvedHonoConnectionTest.java | 3 +- .../pre/ConnectionExistenceChecker.java | 74 ++++++++++ .../ModifyToCreateConnectionTransformer.java | 62 +++++++++ ...PreEnforcementConnectionIdCacheLoader.java | 79 +++++++++++ .../service/enforcement/pre/package-info.java | 14 ++ .../persistence/stages/StagedCommand.java | 17 ++- .../AbstractConnectivityCommandStrategy.java | 15 ++- .../commands/CloseConnectionStrategy.java | 21 ++- .../commands/ConnectionConflictStrategy.java | 18 ++- .../ConnectionUninitializedStrategies.java | 85 ------------ ...nsConditionalHeadersValidatorProvider.java | 83 ++++++++++++ .../commands/CreateConnectionStrategy.java | 44 +++++- .../commands/DeleteConnectionStrategy.java | 22 ++- .../EnableConnectionLogsStrategy.java | 22 ++- .../commands/LoggingExpiredStrategy.java | 21 ++- .../commands/ModifyConnectionStrategy.java | 29 +++- .../commands/OpenConnectionStrategy.java | 20 ++- .../commands/ResetConnectionLogsStrategy.java | 20 ++- .../ResetConnectionMetricsStrategy.java | 23 +++- .../RetrieveConnectionLogsStrategy.java | 22 ++- .../RetrieveConnectionMetricsStrategy.java | 22 ++- .../RetrieveConnectionStatusStrategy.java | 22 ++- .../commands/RetrieveConnectionStrategy.java | 48 ++++++- ...etrieveResolvedHonoConnectionStrategy.java | 47 ++++++- .../commands/StagedCommandStrategy.java | 15 ++- .../SudoAddConnectionLogEntryStrategy.java | 15 +++ .../SudoRetrieveConnectionTagsStrategy.java | 15 +++ .../TestConnectionConflictStrategy.java | 18 ++- .../commands/TestConnectionStrategy.java | 35 ++++- .../events/ConnectionClosedStrategy.java | 4 +- .../events/ConnectionCreatedStrategy.java | 4 +- .../events/ConnectionDeletedStrategy.java | 8 +- .../events/ConnectionModifiedStrategy.java | 4 +- .../events/ConnectionOpenedStrategy.java | 9 +- .../resources/connectivity-extension.conf | 5 + .../messaging/ErrorHandlingActorTest.java | 26 +++- .../ConnectionPersistenceActorTest.java | 101 ++++++++------ .../main/resources/jsonschema/connection.json | 14 ++ .../src/main/resources/jsonschema/policy.json | 76 +++++++---- .../main/resources/jsonschema/thing_v2.json | 22 +-- .../main/resources/openapi/ditto-api-2.yml | 83 +++++++++--- .../resources/openapi/sources/api-2-index.yml | 6 +- .../parameters/connectionFieldsQueryParam.yml | 54 ++++++++ .../sources/paths/connections/command.yml | 2 +- .../paths/connections/connectionId.yml | 7 +- .../sources/paths/connections/connections.yml | 5 +- .../sources/paths/connections/logs.yml | 2 +- .../sources/paths/connections/metrics.yml | 2 +- .../sources/paths/connections/status.yml | 2 +- .../AbstractConnectionsRetrievalActor.java | 14 +- .../connections/ConnectionsParameter.java | 8 +- .../routes/connections/ConnectionsRoute.java | 36 +++-- .../commands/query/RetrievePolicy.java | 2 +- things/service/src/main/resources/things.conf | 2 +- 70 files changed, 1885 insertions(+), 336 deletions(-) create mode 100755 connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionRevision.java create mode 100755 connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableConnectionRevision.java create mode 100644 connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionFailedException.java create mode 100644 connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionNotModifiedException.java create mode 100644 connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ConnectionExistenceChecker.java create mode 100644 connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ModifyToCreateConnectionTransformer.java create mode 100644 connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/PreEnforcementConnectionIdCacheLoader.java create mode 100644 connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/package-info.java delete mode 100644 connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionUninitializedStrategies.java create mode 100644 connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionsConditionalHeadersValidatorProvider.java create mode 100644 documentation/src/main/resources/openapi/sources/parameters/connectionFieldsQueryParam.yml diff --git a/connectivity/model/pom.xml b/connectivity/model/pom.xml index 9ac624708b7..01801773765 100644 --- a/connectivity/model/pom.xml +++ b/connectivity/model/pom.xml @@ -124,7 +124,7 @@ - + org.eclipse.ditto.connectivity.model.Connection#toJson(org.eclipse.ditto.base.model.json.JsonSchemaVersion,org.eclipse.ditto.json.JsonFieldSelector) diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnection.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnection.java index eea09317d52..5a7a284bbfe 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnection.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnection.java @@ -15,6 +15,8 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.text.MessageFormat; +import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -31,6 +33,8 @@ import javax.annotation.Nullable; +import org.eclipse.ditto.base.model.entity.id.EntityId; +import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.json.JsonArray; import org.eclipse.ditto.json.JsonCollectors; @@ -56,6 +60,9 @@ abstract class AbstractConnection implements Connection { @Nullable private final Credentials credentials; @Nullable private final String trustedCertificates; @Nullable private final ConnectionLifecycle lifecycle; + @Nullable private final ConnectionRevision revision; + @Nullable private final Instant modified; + @Nullable private final Instant created; private final List sources; private final List targets; private final int clientCount; @@ -85,12 +92,15 @@ abstract class AbstractConnection implements Connection { payloadMappingDefinition = builder.payloadMappingDefinition; tags = Collections.unmodifiableSet(new LinkedHashSet<>(builder.tags)); lifecycle = builder.lifecycle; + revision = builder.revision; + modified = builder.modified; + created = builder.created; sshTunnel = builder.sshTunnel; } abstract ConnectionUri getConnectionUri(@Nullable String builderConnectionUri); - static void buildFromJson (final JsonObject jsonObject, final AbstractConnectionBuilder builder) { + static void buildFromJson(final JsonObject jsonObject, final AbstractConnectionBuilder builder) { final MappingContext mappingContext = jsonObject.getValue(JsonFields.MAPPING_CONTEXT) .map(ConnectivityModelFactory::mappingContextFromJson) .orElse(null); @@ -112,6 +122,12 @@ static void buildFromJson (final JsonObject jsonObject, final AbstractConnection jsonObject.getValue(JsonFields.LIFECYCLE) .flatMap(ConnectionLifecycle::forName).ifPresent(builder::lifecycle); + jsonObject.getValue(JsonFields.REVISION) + .map(ConnectionRevision::newInstance).ifPresent(builder::revision); + jsonObject.getValue(JsonFields.MODIFIED) + .map(AbstractConnection::tryToParseInstant).ifPresent(builder::modified); + jsonObject.getValue(JsonFields.CREATED) + .map(AbstractConnection::tryToParseInstant).ifPresent(builder::created); jsonObject.getValue(JsonFields.CREDENTIALS).ifPresent(builder::credentialsFromJson); jsonObject.getValue(JsonFields.CLIENT_COUNT).ifPresent(builder::clientCount); jsonObject.getValue(JsonFields.FAILOVER_ENABLED).ifPresent(builder::failoverEnabled); @@ -296,6 +312,36 @@ public Optional getLifecycle() { return Optional.ofNullable(lifecycle); } + @Override + public Optional getEntityId() { + return Optional.of(id); + } + + @Override + public Optional getRevision() { + return Optional.ofNullable(revision); + } + + @Override + public Optional getModified() { + return Optional.ofNullable(modified); + } + + @Override + public Optional getCreated() { + return Optional.ofNullable(created); + } + + @Override + public Optional getMetadata() { + return Optional.empty(); // currently not metadata support for connections + } + + @Override + public boolean isDeleted() { + return ConnectionLifecycle.DELETED.equals(lifecycle); + } + static ConnectionBuilder fromConnection(final Connection connection, final AbstractConnectionBuilder builder) { checkNotNull(connection, "Connection"); @@ -316,7 +362,10 @@ static ConnectionBuilder fromConnection(final Connection connection, final Abstr .name(connection.getName().orElse(null)) .sshTunnel(connection.getSshTunnel().orElse(null)) .tags(connection.getTags()) - .lifecycle(connection.getLifecycle().orElse(null)); + .lifecycle(connection.getLifecycle().orElse(null)) + .revision(connection.getRevision().orElse(null)) + .modified(connection.getModified().orElse(null)) + .created(connection.getCreated().orElse(null)); } @Override @@ -329,6 +378,15 @@ public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate< } jsonObjectBuilder.set(JsonFields.ID, String.valueOf(id), predicate); jsonObjectBuilder.set(JsonFields.NAME, name, predicate); + if (null != revision) { + jsonObjectBuilder.set(JsonFields.REVISION, revision.toLong(), predicate); + } + if (null != modified) { + jsonObjectBuilder.set(JsonFields.MODIFIED, modified.toString(), predicate); + } + if (null != created) { + jsonObjectBuilder.set(JsonFields.CREATED, created.toString(), predicate); + } jsonObjectBuilder.set(JsonFields.CONNECTION_TYPE, connectionType.getName(), predicate); jsonObjectBuilder.set(JsonFields.CONNECTION_STATUS, connectionStatus.getName(), predicate); jsonObjectBuilder.set(JsonFields.URI, uri.toString(), predicate); @@ -368,6 +426,15 @@ public JsonObject toJson(final JsonSchemaVersion schemaVersion, final Predicate< return jsonObjectBuilder.build(); } + private static Instant tryToParseInstant(final CharSequence dateTime) { + try { + return Instant.parse(dateTime); + } catch (final DateTimeParseException e) { + throw new JsonParseException("The JSON object's field '" + Connection.JsonFields.MODIFIED.getPointer() + "' " + + "is not in ISO-8601 format as expected"); + } + } + @SuppressWarnings("OverlyComplexMethod") @Override public boolean equals(@Nullable final Object o) { @@ -394,6 +461,9 @@ public boolean equals(@Nullable final Object o) { Objects.equals(specificConfig, that.specificConfig) && Objects.equals(payloadMappingDefinition, that.payloadMappingDefinition) && Objects.equals(lifecycle, that.lifecycle) && + Objects.equals(revision, that.revision) && + Objects.equals(modified, that.modified) && + Objects.equals(created, that.created) && Objects.equals(sshTunnel, that.sshTunnel) && Objects.equals(tags, that.tags); } @@ -402,7 +472,7 @@ public boolean equals(@Nullable final Object o) { public int hashCode() { return Objects.hash(id, name, connectionType, connectionStatus, sources, targets, clientCount, failOverEnabled, credentials, trustedCertificates, uri, validateCertificate, processorPoolSize, specificConfig, - payloadMappingDefinition, sshTunnel, tags, lifecycle); + payloadMappingDefinition, sshTunnel, tags, lifecycle, revision, modified, created); } @Override @@ -426,6 +496,9 @@ public String toString() { ", payloadMappingDefinition=" + payloadMappingDefinition + ", tags=" + tags + ", lifecycle=" + lifecycle + + ", revision=" + revision + + ", modified=" + modified + + ", created=" + created + "]"; } diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnectionBuilder.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnectionBuilder.java index 5bf00d4797a..6243c22352f 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnectionBuilder.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/AbstractConnectionBuilder.java @@ -16,6 +16,7 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.text.MessageFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -49,6 +50,9 @@ abstract class AbstractConnectionBuilder implements ConnectionBuilder { @Nullable MappingContext mappingContext = null; @Nullable String trustedCertificates; @Nullable ConnectionLifecycle lifecycle = null; + @Nullable ConnectionRevision revision = null; + @Nullable Instant modified = null; + @Nullable Instant created = null; @Nullable SshTunnel sshTunnel = null; // optional with default: @@ -192,6 +196,24 @@ public ConnectionBuilder lifecycle(@Nullable final ConnectionLifecycle lifecycle return this; } + @Override + public ConnectionBuilder revision(@Nullable final ConnectionRevision revision) { + this.revision = revision; + return this; + } + + @Override + public ConnectionBuilder modified(@Nullable final Instant modified) { + this.modified = modified; + return this; + } + + @Override + public ConnectionBuilder created(@Nullable final Instant created) { + this.created = created; + return this; + } + @Override public ConnectionBuilder sshTunnel(@Nullable final SshTunnel sshTunnel) { this.sshTunnel = sshTunnel; diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/Connection.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/Connection.java index b9a99b60f1e..bcb51110a54 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/Connection.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/Connection.java @@ -20,21 +20,19 @@ import javax.annotation.concurrent.Immutable; +import org.eclipse.ditto.base.model.entity.Entity; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.base.model.json.Jsonifiable; import org.eclipse.ditto.json.JsonArray; import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonField; import org.eclipse.ditto.json.JsonFieldDefinition; -import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; /** * Represents a connection within the Connectivity service. */ @Immutable -public interface Connection extends Jsonifiable.WithFieldSelectorAndPredicate { +public interface Connection extends Entity { /** * Returns the identifier of this {@code Connection}. @@ -242,11 +240,6 @@ default JsonObject toJson() { return toJson(FieldType.notHidden()); } - @Override - default JsonObject toJson(final JsonSchemaVersion schemaVersion, final JsonFieldSelector fieldSelector) { - return toJson(schemaVersion, FieldType.notHidden()).get(fieldSelector); - } - /** * An enumeration of the known {@code JsonField}s of a {@code Connection}. */ @@ -261,6 +254,33 @@ final class JsonFields { FieldType.HIDDEN, JsonSchemaVersion.V_2); + /** + * JSON field containing the Connection's revision. + * @since 3.2.0 + */ + public static final JsonFieldDefinition REVISION = JsonFactory.newLongFieldDefinition("_revision", + FieldType.SPECIAL, + FieldType.HIDDEN, + JsonSchemaVersion.V_2); + + /** + * JSON field containing the Connection's modified timestamp in ISO-8601 format. + * @since 3.2.0 + */ + public static final JsonFieldDefinition MODIFIED = JsonFactory.newStringFieldDefinition("_modified", + FieldType.SPECIAL, + FieldType.HIDDEN, + JsonSchemaVersion.V_2); + + /** + * JSON field containing the Connection's created timestamp in ISO-8601 format. + * @since 3.2.0 + */ + public static final JsonFieldDefinition CREATED = JsonFactory.newStringFieldDefinition("_created", + FieldType.SPECIAL, + FieldType.HIDDEN, + JsonSchemaVersion.V_2); + /** * JSON field containing the {@code Connection} identifier. */ diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionBuilder.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionBuilder.java index a124f64ccaa..56daf691b7b 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionBuilder.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionBuilder.java @@ -12,6 +12,7 @@ */ package org.eclipse.ditto.connectivity.model; +import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; @@ -203,6 +204,44 @@ default ConnectionBuilder credentialsFromJson(final JsonObject jsonObject) { */ ConnectionBuilder lifecycle(@Nullable ConnectionLifecycle lifecycle); + /** + * Sets the given revision number to this builder. + * + * @param revisionNumber the revision number to be set. + * @return this builder to allow method chaining. + * @since 3.2.0 + */ + default ConnectionBuilder revision(final long revisionNumber) { + return revision(ConnectionRevision.newInstance(revisionNumber)); + } + + /** + * Sets the {@link ConnectionRevision} of the connection. + * + * @param revision the connection revision + * @return this builder + * @since 3.2.0 + */ + ConnectionBuilder revision(@Nullable ConnectionRevision revision); + + /** + * Sets the given modified timestamp to this builder. + * + * @param modified the timestamp to be set. + * @return this builder to allow method chaining. + * @since 3.2.0 + */ + ConnectionBuilder modified(@Nullable Instant modified); + + /** + * Sets the given created timestamp to this builder. + * + * @param created the created timestamp to be set. + * @return this builder to allow method chaining. + * @since 3.2.0 + */ + ConnectionBuilder created(@Nullable Instant created); + /** * Sets the {@link SshTunnel} of the connection. * diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionRevision.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionRevision.java new file mode 100755 index 00000000000..2986f7a82d3 --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectionRevision.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model; + +import org.eclipse.ditto.base.model.entity.Revision; + +/** + * Represents the current revision of a Connection. + */ +public interface ConnectionRevision extends Revision { + + /** + * Returns a new immutable {@code ConnectionRevision} which is initialised with the given revision number. + * + * @param revisionNumber the {@code long} value of the revision. + * @return the new immutable {@code ConnectionRevision}. + */ + static ConnectionRevision newInstance(final long revisionNumber) { + return ConnectivityModelFactory.newConnectionRevision(revisionNumber); + } + +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectivityModelFactory.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectivityModelFactory.java index 61ca499af91..4ace7c5435f 100755 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectivityModelFactory.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ConnectivityModelFactory.java @@ -991,4 +991,15 @@ public static LogEntryBuilder newLogEntryBuilder(final String correlationId, return ImmutableLogEntry.getBuilder(correlationId, timestamp, logCategory, logType, logLevel, message); } + /** + * Returns a new immutable {@link ConnectionRevision} which is initialised with the given revision number. + * + * @param revisionNumber the {@code long} value of the revision. + * @return the new immutable {@code ConnectionRevision}. + * @since 3.2.0 + */ + public static ConnectionRevision newConnectionRevision(final long revisionNumber) { + return ImmutableConnectionRevision.of(revisionNumber); + } + } diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableConnectionRevision.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableConnectionRevision.java new file mode 100755 index 00000000000..c9ec22b8000 --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/ImmutableConnectionRevision.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model; + +import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; + +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +/** + * An immutable implementation of {@link ConnectionRevision}. + */ +@Immutable +final class ImmutableConnectionRevision implements ConnectionRevision { + + private final long value; + + private ImmutableConnectionRevision(final long theValue) { + value = theValue; + } + + /** + * Returns a new instance of {@code ConnectionRevision} with the given value. + * + * @param value the value of the new revision. + * @return a new Connection revision. + */ + public static ImmutableConnectionRevision of(final long value) { + return new ImmutableConnectionRevision(value); + } + + @Override + public boolean isGreaterThan(final ConnectionRevision other) { + return 0 < compareTo(other); + } + + @Override + public boolean isGreaterThanOrEqualTo(final ConnectionRevision other) { + return 0 <= compareTo(other); + } + + @Override + public boolean isLowerThan(final ConnectionRevision other) { + return 0 > compareTo(other); + } + + @Override + public boolean isLowerThanOrEqualTo(final ConnectionRevision other) { + return 0 >= compareTo(other); + } + + @Override + public ConnectionRevision increment() { + return of(value + 1); + } + + @Override + public long toLong() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final ImmutableConnectionRevision that = (ImmutableConnectionRevision) o; + return value == that.value; + } + + @Override + public int hashCode() { + return Objects.hash(value); + } + + @Override + public int compareTo(final ConnectionRevision o) { + checkNotNull(o, "other revision to compare this revision with"); + return Long.compare(value, o.toLong()); + } + +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java index 6c33196e410..53ead396749 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java @@ -12,14 +12,16 @@ */ package org.eclipse.ditto.connectivity.model.signals.commands; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonFieldDefinition; -import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.base.model.entity.type.EntityType; +import org.eclipse.ditto.base.model.entity.type.WithEntityType; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.connectivity.model.ConnectivityConstants; import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonPointer; /** * Base interface for all commands which are understood by the Connectivity service. Implementations of this interface @@ -27,7 +29,8 @@ * * @param the type of the implementing class. */ -public interface ConnectivityCommand> extends Command { +public interface ConnectivityCommand> extends Command, + WithEntityType { /** * Type Prefix of Connectivity commands. @@ -57,6 +60,17 @@ default String getTypePrefix() { @Override T setDittoHeaders(DittoHeaders dittoHeaders); + /** + * Returns the entity type {@link ConnectivityConstants#ENTITY_TYPE}. + * + * @return the Connection entity type. + * @since 3.2.0 + */ + @Override + default EntityType getEntityType() { + return ConnectivityConstants.ENTITY_TYPE; + } + /** * This class contains definitions for all specific fields of a {@code ConnectivityCommand}'s JSON representation. */ diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionFailedException.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionFailedException.java new file mode 100644 index 00000000000..6746f57942c --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionFailedException.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model.signals.commands.exceptions; + +import java.net.URI; +import java.text.MessageFormat; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.connectivity.model.ConnectivityException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Thrown when validating a precondition header fails on a Connection or one of its sub-entities. + * @since 3.2.0 + */ +@Immutable +@JsonParsableException(errorCode = ConnectionPreconditionFailedException.ERROR_CODE) +public final class ConnectionPreconditionFailedException extends DittoRuntimeException implements + ConnectivityException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "precondition.failed"; + + private static final String MESSAGE_TEMPLATE = + "The comparison of precondition header ''{0}'' for the requested connection resource evaluated to false." + + " Header value: ''{1}'', actual entity-tag: ''{2}''."; + + private static final String DEFAULT_DESCRIPTION = "The comparison of the provided precondition header with the " + + "current ETag value of the requested connection resource evaluated to false. Check the value of your " + + "conditional header value."; + + private ConnectionPreconditionFailedException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + super(ERROR_CODE, HttpStatus.PRECONDITION_FAILED, dittoHeaders, message, description, cause, href); + } + + /** + * A mutable builder for a {@link ConnectionPreconditionFailedException}. + * + * @param conditionalHeaderName the name of the conditional header. + * @param expected the expected value. + * @param actual the actual ETag value. + * @return the builder. + */ + public static Builder newBuilder(final String conditionalHeaderName, final String expected, final String actual) { + return new Builder(conditionalHeaderName, expected, actual); + } + + /** + * Constructs a new {@link ConnectionPreconditionFailedException} object with the exception message extracted from + * the given JSON object. + * + * @param jsonObject the JSON to read the {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new {@link ConnectionPreconditionFailedException}. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static ConnectionPreconditionFailedException fromJson(final JsonObject jsonObject, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link ConnectionPreconditionFailedException}. + */ + @NotThreadSafe + public static final class Builder + extends DittoRuntimeExceptionBuilder { + + private Builder() { + description(DEFAULT_DESCRIPTION); + } + + private Builder(final String conditionalHeaderName, final String expected, final String actual) { + this(); + message(MessageFormat.format(MESSAGE_TEMPLATE, conditionalHeaderName, expected, actual)); + } + + @Override + protected ConnectionPreconditionFailedException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new ConnectionPreconditionFailedException(dittoHeaders, message, description, cause, href); + } + + } +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionNotModifiedException.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionNotModifiedException.java new file mode 100644 index 00000000000..ecb5143274b --- /dev/null +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/exceptions/ConnectionPreconditionNotModifiedException.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.model.signals.commands.exceptions; + +import java.net.URI; +import java.text.MessageFormat; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.NotThreadSafe; + +import org.eclipse.ditto.base.model.common.HttpStatus; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.JsonParsableException; +import org.eclipse.ditto.connectivity.model.ConnectivityException; +import org.eclipse.ditto.json.JsonObject; + +/** + * Thrown when validating a precondition header on a connection or one of its sub-entities leads to status + * {@link org.eclipse.ditto.base.model.common.HttpStatus#NOT_MODIFIED}. + */ +@Immutable +@JsonParsableException(errorCode = ConnectionPreconditionNotModifiedException.ERROR_CODE) +public final class ConnectionPreconditionNotModifiedException extends DittoRuntimeException + implements ConnectivityException { + + /** + * Error code of this exception. + */ + public static final String ERROR_CODE = ERROR_CODE_PREFIX + "precondition.notmodified"; + + private static final String MESSAGE_TEMPLATE = + "The comparison of precondition header ''if-none-match'' for the requested connection resource evaluated to " + + "false. Expected: ''{0}'' not to match actual: ''{1}''."; + + private static final String DEFAULT_DESCRIPTION = + "The comparison of the provided precondition header ''if-none-match'' with the current ETag value of the " + + "requested connection resource evaluated to false. Check the value of your conditional header value."; + + private ConnectionPreconditionNotModifiedException(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + super(ERROR_CODE, HttpStatus.NOT_MODIFIED, dittoHeaders, message, description, cause, href); + } + + /** + * A mutable builder for a {@link ConnectionPreconditionNotModifiedException}. + * + * @param expectedNotToMatch the value which was expected not to match {@code matched} value. + * @param matched the matched value. + * @return the builder. + */ + public static Builder newBuilder(final String expectedNotToMatch, final String matched) { + return new Builder(expectedNotToMatch, matched); + } + + /** + * Constructs a new {@link ConnectionPreconditionNotModifiedException} object with the exception message extracted from + * the given JSON object. + * + * @param jsonObject the JSON to read the + * {@link org.eclipse.ditto.base.model.exceptions.DittoRuntimeException.JsonFields#MESSAGE} field from. + * @param dittoHeaders the headers of the command which resulted in this exception. + * @return the new ConditionalHeadersNotModifiedException. + * @throws NullPointerException if any argument is {@code null}. + * @throws org.eclipse.ditto.json.JsonMissingFieldException if this JsonObject did not contain an error message. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static ConnectionPreconditionNotModifiedException fromJson(final JsonObject jsonObject, + final DittoHeaders dittoHeaders) { + return DittoRuntimeException.fromJson(jsonObject, dittoHeaders, new Builder()); + } + + @Override + public DittoRuntimeException setDittoHeaders(final DittoHeaders dittoHeaders) { + return new Builder() + .message(getMessage()) + .description(getDescription().orElse(null)) + .cause(getCause()) + .href(getHref().orElse(null)) + .dittoHeaders(dittoHeaders) + .build(); + } + + /** + * A mutable builder with a fluent API for a {@link ConnectionPreconditionNotModifiedException}. + */ + @NotThreadSafe + public static final class Builder extends DittoRuntimeExceptionBuilder { + + private Builder() { + description(DEFAULT_DESCRIPTION); + } + + private Builder(final String expectedNotToMatch, final String matched) { + this(); + message(MessageFormat.format(MESSAGE_TEMPLATE, expectedNotToMatch, matched)); + } + + @Override + protected ConnectionPreconditionNotModifiedException doBuild(final DittoHeaders dittoHeaders, + @Nullable final String message, + @Nullable final String description, + @Nullable final Throwable cause, + @Nullable final URI href) { + return new ConnectionPreconditionNotModifiedException(dittoHeaders, message, description, cause, href); + } + + } +} diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnection.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnection.java index bd9e0c25df1..dfafac737c3 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnection.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnection.java @@ -15,24 +15,29 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.util.Objects; +import java.util.Optional; import java.util.function.Predicate; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; -import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonField; -import org.eclipse.ditto.json.JsonObject; -import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonParsableCommand; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; -import org.eclipse.ditto.connectivity.model.ConnectionId; -import org.eclipse.ditto.connectivity.model.WithConnectionId; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.AbstractCommand; import org.eclipse.ditto.base.model.signals.commands.CommandJsonDeserializer; +import org.eclipse.ditto.base.model.signals.commands.WithSelectedFields; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.WithConnectionId; +import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonFieldSelector; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; /** * Command which retrieves a {@link org.eclipse.ditto.connectivity.model.Connection}. @@ -40,7 +45,8 @@ @Immutable @JsonParsableCommand(typePrefix = ConnectivityCommand.TYPE_PREFIX, name = RetrieveConnection.NAME) public final class RetrieveConnection extends AbstractCommand - implements ConnectivityQueryCommand, WithConnectionId, SignalWithEntityId { + implements ConnectivityQueryCommand, WithConnectionId, WithSelectedFields, + SignalWithEntityId { /** * Name of this command. @@ -52,11 +58,19 @@ public final class RetrieveConnection extends AbstractCommand JSON_SELECTED_FIELDS = + JsonFactory.newStringFieldDefinition("selectedFields", FieldType.REGULAR, + JsonSchemaVersion.V_2); + private final ConnectionId connectionId; + @Nullable private final JsonFieldSelector selectedFields; - private RetrieveConnection(final ConnectionId connectionId, final DittoHeaders dittoHeaders) { + private RetrieveConnection(final ConnectionId connectionId, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { super(TYPE, dittoHeaders); this.connectionId = connectionId; + this.selectedFields = selectedFields; } /** @@ -69,7 +83,23 @@ private RetrieveConnection(final ConnectionId connectionId, final DittoHeaders d */ public static RetrieveConnection of(final ConnectionId connectionId, final DittoHeaders dittoHeaders) { checkNotNull(connectionId, "Connection ID"); - return new RetrieveConnection(connectionId, dittoHeaders); + return new RetrieveConnection(connectionId, null, dittoHeaders); + } + + /** + * Returns a new instance of {@code RetrieveConnection}. + * + * @param connectionId the identifier of the connection to be retrieved. + * @param selectedFields the fields of the JSON representation of the Connection to retrieve. + * @param dittoHeaders the headers of the request. + * @return a new RetrieveConnection command. + * @throws NullPointerException if any argument is {@code null}. + */ + public static RetrieveConnection of(final ConnectionId connectionId, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { + checkNotNull(connectionId, "Connection ID"); + return new RetrieveConnection(connectionId, selectedFields, dittoHeaders); } /** @@ -101,8 +131,12 @@ public static RetrieveConnection fromJson(final JsonObject jsonObject, final Dit return new CommandJsonDeserializer(TYPE, jsonObject).deserialize(() -> { final String readConnectionId = jsonObject.getValueOrThrow(ConnectivityCommand.JsonFields.JSON_CONNECTION_ID); final ConnectionId connectionId = ConnectionId.of(readConnectionId); + final Optional selectedFields = jsonObject.getValue(JSON_SELECTED_FIELDS) + .map(str -> JsonFactory.newFieldSelector(str, JsonFactory.newParseOptionsBuilder() + .withoutUrlDecoding() + .build())); - return of(connectionId, dittoHeaders); + return of(connectionId, selectedFields.orElse(null), dittoHeaders); }); } @@ -113,6 +147,9 @@ protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final Js final Predicate predicate = schemaVersion.and(thePredicate); jsonObjectBuilder.set(ConnectivityCommand.JsonFields.JSON_CONNECTION_ID, String.valueOf(connectionId), predicate); + if (null != selectedFields) { + jsonObjectBuilder.set(JSON_SELECTED_FIELDS, selectedFields.toString(), predicate); + } } @Override @@ -125,9 +162,14 @@ public Category getCategory() { return Category.QUERY; } + @Override + public Optional getSelectedFields() { + return Optional.ofNullable(selectedFields); + } + @Override public RetrieveConnection setDittoHeaders(final DittoHeaders dittoHeaders) { - return of(connectionId, dittoHeaders); + return of(connectionId, selectedFields, dittoHeaders); } @Override @@ -147,12 +189,13 @@ public boolean equals(@Nullable final Object o) { return false; } final RetrieveConnection that = (RetrieveConnection) o; - return Objects.equals(connectionId, that.connectionId); + return Objects.equals(connectionId, that.connectionId) && + Objects.equals(selectedFields, that.selectedFields); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), connectionId); + return Objects.hash(super.hashCode(), connectionId, selectedFields); } @Override @@ -160,6 +203,7 @@ public String toString() { return getClass().getSimpleName() + " [" + super.toString() + ", connectionId=" + connectionId + + ", selectedFields=" + selectedFields + "]"; } diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnections.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnections.java index 319d1edd04b..aab45f4854f 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnections.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnections.java @@ -13,6 +13,7 @@ package org.eclipse.ditto.connectivity.model.signals.commands.query; import java.util.Objects; +import java.util.Optional; import java.util.function.Predicate; import javax.annotation.Nullable; @@ -23,10 +24,12 @@ import org.eclipse.ditto.base.model.json.JsonParsableCommand; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.commands.AbstractCommand; +import org.eclipse.ditto.base.model.signals.commands.WithSelectedFields; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonField; import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.json.JsonPointer; @@ -40,7 +43,7 @@ @Immutable @JsonParsableCommand(typePrefix = ConnectivityCommand.TYPE_PREFIX, name = RetrieveConnections.NAME) public final class RetrieveConnections extends AbstractCommand - implements ConnectivityQueryCommand { + implements ConnectivityQueryCommand, WithSelectedFields { /** * Name of the "Retrieve Connections" command. @@ -54,11 +57,19 @@ public final class RetrieveConnections extends AbstractCommand JSON_IDS_ONLY = JsonFactory.newBooleanFieldDefinition("idsOnly", FieldType.REGULAR, JsonSchemaVersion.V_2); + static final JsonFieldDefinition JSON_SELECTED_FIELDS = + JsonFactory.newStringFieldDefinition("selectedFields", FieldType.REGULAR, + JsonSchemaVersion.V_2); + private final boolean idsOnly; + @Nullable private final JsonFieldSelector selectedFields; - private RetrieveConnections(final boolean idsOnly, final DittoHeaders dittoHeaders) { + private RetrieveConnections(final boolean idsOnly, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { super(TYPE, dittoHeaders); this.idsOnly = idsOnly; + this.selectedFields = selectedFields; } /** @@ -70,7 +81,22 @@ private RetrieveConnections(final boolean idsOnly, final DittoHeaders dittoHeade */ public static RetrieveConnections newInstance(final boolean idsOnly, final DittoHeaders dittoHeaders) { - return new RetrieveConnections(idsOnly, dittoHeaders); + return new RetrieveConnections(idsOnly, null, dittoHeaders); + } + + /** + * Returns a new instance of the retrieve connections command. + * + * @param dittoHeaders provide additional information regarding connections retrieval like a correlation ID. + * @param selectedFields the fields of the JSON representation of the Connection to retrieve. + * @return the instance. + * @throws NullPointerException if any argument is {@code null}. + */ + public static RetrieveConnections newInstance(final boolean idsOnly, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { + + return new RetrieveConnections(idsOnly, selectedFields, dittoHeaders); } /** @@ -100,8 +126,13 @@ public static RetrieveConnections fromJson(final String jsonString, final DittoH */ public static RetrieveConnections fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { final boolean idsOnly = jsonObject.getValueOrThrow(JSON_IDS_ONLY); + final JsonFieldSelector extractedFieldSelector = jsonObject.getValue(JSON_SELECTED_FIELDS) + .map(str -> JsonFactory.newFieldSelector(str, JsonFactory.newParseOptionsBuilder() + .withoutUrlDecoding() + .build())) + .orElse(null); - return new RetrieveConnections(idsOnly, dittoHeaders); + return new RetrieveConnections(idsOnly, extractedFieldSelector, dittoHeaders); } /** @@ -113,6 +144,11 @@ public boolean getIdsOnly() { return idsOnly; } + @Override + public Optional getSelectedFields() { + return Optional.ofNullable(selectedFields); + } + @Override public Category getCategory() { return Category.QUERY; @@ -120,7 +156,7 @@ public Category getCategory() { @Override public RetrieveConnections setDittoHeaders(final DittoHeaders dittoHeaders) { - return new RetrieveConnections(idsOnly, dittoHeaders); + return new RetrieveConnections(idsOnly, selectedFields, dittoHeaders); } @Override @@ -134,6 +170,9 @@ protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final Js final Predicate predicate = jsonSchemaVersion.and(thePredicate); jsonObjectBuilder.set(JSON_IDS_ONLY, idsOnly, predicate); + if (null != selectedFields) { + jsonObjectBuilder.set(JSON_SELECTED_FIELDS, selectedFields.toString(), predicate); + } } @Override @@ -148,12 +187,13 @@ public boolean equals(@Nullable final Object o) { return false; } final RetrieveConnections that = (RetrieveConnections) o; - return Objects.equals(idsOnly, that.idsOnly); + return Objects.equals(idsOnly, that.idsOnly) && + Objects.equals(selectedFields, that.selectedFields); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), idsOnly); + return Objects.hash(super.hashCode(), idsOnly, selectedFields); } @Override @@ -161,6 +201,7 @@ public String toString() { return getClass().getSimpleName() + " [" + super.toString() + ", idsOnly=" + idsOnly + + ", selectedFields=" + selectedFields + "]"; } diff --git a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnection.java b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnection.java index f4ba9a53678..54441b71ece 100644 --- a/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnection.java +++ b/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnection.java @@ -15,22 +15,27 @@ import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull; import java.util.Objects; +import java.util.Optional; import java.util.function.Predicate; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.json.FieldType; import org.eclipse.ditto.base.model.json.JsonParsableCommand; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; import org.eclipse.ditto.base.model.signals.SignalWithEntityId; import org.eclipse.ditto.base.model.signals.commands.AbstractCommand; import org.eclipse.ditto.base.model.signals.commands.CommandJsonDeserializer; +import org.eclipse.ditto.base.model.signals.commands.WithSelectedFields; import org.eclipse.ditto.connectivity.model.ConnectionId; import org.eclipse.ditto.connectivity.model.WithConnectionId; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonObjectBuilder; @@ -43,7 +48,8 @@ @Immutable @JsonParsableCommand(typePrefix = ConnectivityCommand.TYPE_PREFIX, name = RetrieveResolvedHonoConnection.NAME) public final class RetrieveResolvedHonoConnection extends AbstractCommand - implements ConnectivityQueryCommand, WithConnectionId, SignalWithEntityId { + implements ConnectivityQueryCommand, WithConnectionId, WithSelectedFields, + SignalWithEntityId { /** * Name of this command. @@ -55,11 +61,19 @@ public final class RetrieveResolvedHonoConnection extends AbstractCommand JSON_SELECTED_FIELDS = + JsonFactory.newStringFieldDefinition("selectedFields", FieldType.REGULAR, + JsonSchemaVersion.V_2); + private final ConnectionId connectionId; + @Nullable private final JsonFieldSelector selectedFields; - private RetrieveResolvedHonoConnection(final ConnectionId connectionId, final DittoHeaders dittoHeaders) { + private RetrieveResolvedHonoConnection(final ConnectionId connectionId, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { super(TYPE, dittoHeaders); this.connectionId = connectionId; + this.selectedFields = selectedFields; } /** @@ -72,7 +86,23 @@ private RetrieveResolvedHonoConnection(final ConnectionId connectionId, final Di */ public static RetrieveResolvedHonoConnection of(final ConnectionId connectionId, final DittoHeaders dittoHeaders) { checkNotNull(connectionId, "connectionId"); - return new RetrieveResolvedHonoConnection(connectionId, dittoHeaders); + return new RetrieveResolvedHonoConnection(connectionId, null, dittoHeaders); + } + + /** + * Returns a new instance of {@code RetrieveResolvedHonoConnection}. + * + * @param connectionId the identifier of the connection to be retrieved. + * @param selectedFields the fields of the JSON representation of the HonoConnection to retrieve. + * @param dittoHeaders the headers of the request. + * @return a new RetrieveResolvedHonoConnection command. + * @throws NullPointerException if any argument is {@code null}. + */ + public static RetrieveResolvedHonoConnection of(final ConnectionId connectionId, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { + checkNotNull(connectionId, "Connection ID"); + return new RetrieveResolvedHonoConnection(connectionId, selectedFields, dittoHeaders); } /** @@ -104,8 +134,12 @@ public static RetrieveResolvedHonoConnection fromJson(final JsonObject jsonObjec return new CommandJsonDeserializer(TYPE, jsonObject).deserialize(() -> { final String readConnectionId = jsonObject.getValueOrThrow(ConnectivityCommand.JsonFields.JSON_CONNECTION_ID); final ConnectionId connectionId = ConnectionId.of(readConnectionId); + final Optional selectedFields = jsonObject.getValue(JSON_SELECTED_FIELDS) + .map(str -> JsonFactory.newFieldSelector(str, JsonFactory.newParseOptionsBuilder() + .withoutUrlDecoding() + .build())); - return of(connectionId, dittoHeaders); + return of(connectionId, selectedFields.orElse(null), dittoHeaders); }); } @@ -116,6 +150,9 @@ protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final Js final Predicate predicate = schemaVersion.and(thePredicate); jsonObjectBuilder.set(ConnectivityCommand.JsonFields.JSON_CONNECTION_ID, String.valueOf(connectionId), predicate); + if (null != selectedFields) { + jsonObjectBuilder.set(JSON_SELECTED_FIELDS, selectedFields.toString(), predicate); + } } @Override @@ -128,9 +165,14 @@ public Category getCategory() { return Category.QUERY; } + @Override + public Optional getSelectedFields() { + return Optional.ofNullable(selectedFields); + } + @Override public RetrieveResolvedHonoConnection setDittoHeaders(final DittoHeaders dittoHeaders) { - return of(connectionId, dittoHeaders); + return of(connectionId, selectedFields, dittoHeaders); } @Override @@ -150,12 +192,13 @@ public boolean equals(@Nullable final Object o) { return false; } final RetrieveResolvedHonoConnection that = (RetrieveResolvedHonoConnection) o; - return Objects.equals(connectionId, that.connectionId); + return Objects.equals(connectionId, that.connectionId) && + Objects.equals(selectedFields, that.selectedFields); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), connectionId); + return Objects.hash(super.hashCode(), connectionId, selectedFields); } @Override @@ -163,6 +206,7 @@ public String toString() { return getClass().getSimpleName() + " [" + super.toString() + ", connectionId=" + connectionId + + ", selectedFields=" + selectedFields + "]"; } diff --git a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionTest.java b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionTest.java index 1137132cc16..eca9da01802 100644 --- a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionTest.java +++ b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionTest.java @@ -24,6 +24,7 @@ import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.connectivity.model.signals.commands.TestConstants; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.json.assertions.DittoJsonAssertions; @@ -52,7 +53,7 @@ public void testHashCodeAndEquals() { public void assertImmutability() { assertInstancesOf(RetrieveConnection.class, areImmutable(), - provided(ConnectionId.class).isAlsoImmutable()); + provided(ConnectionId.class, JsonFieldSelector.class).isAlsoImmutable()); } @Test diff --git a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionsTest.java b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionsTest.java index 375d243bf80..8d70bccc83f 100644 --- a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionsTest.java +++ b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveConnectionsTest.java @@ -15,6 +15,7 @@ import static org.eclipse.ditto.json.assertions.DittoJsonAssertions.assertThat; import static org.mutabilitydetector.unittesting.AllowedReason.assumingFields; +import static org.mutabilitydetector.unittesting.AllowedReason.provided; import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; @@ -28,6 +29,7 @@ import org.eclipse.ditto.base.model.signals.commands.GlobalCommandRegistry; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.json.assertions.DittoJsonAssertions; @@ -72,7 +74,8 @@ public void setUp() { public void assertImmutability() { assertInstancesOf(RetrieveConnections.class, areImmutable(), - assumingFields("connectionIds").areSafelyCopiedUnmodifiableCollectionsWithImmutableElements()); + assumingFields("connectionIds").areSafelyCopiedUnmodifiableCollectionsWithImmutableElements(), + provided(JsonFieldSelector.class).isAlsoImmutable()); } @Test diff --git a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnectionTest.java b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnectionTest.java index 94bd12fb22a..5018d59d8af 100644 --- a/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnectionTest.java +++ b/connectivity/model/src/test/java/org/eclipse/ditto/connectivity/model/signals/commands/query/RetrieveResolvedHonoConnectionTest.java @@ -27,6 +27,7 @@ import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.connectivity.model.signals.commands.TestConstants; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonPointer; import org.eclipse.ditto.json.assertions.DittoJsonAssertions; @@ -55,7 +56,7 @@ public void testHashCodeAndEquals() { public void assertImmutability() { assertInstancesOf(RetrieveResolvedHonoConnection.class, areImmutable(), - provided(ConnectionId.class).isAlsoImmutable()); + provided(ConnectionId.class, JsonFieldSelector.class).isAlsoImmutable()); } @Test diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ConnectionExistenceChecker.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ConnectionExistenceChecker.java new file mode 100644 index 00000000000..24fa1f91095 --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ConnectionExistenceChecker.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.service.enforcement.pre; + +import java.util.concurrent.CompletionStage; + +import org.eclipse.ditto.connectivity.api.ConnectivityMessagingConstants; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; +import org.eclipse.ditto.internal.utils.cache.entry.Entry; +import org.eclipse.ditto.internal.utils.cluster.ShardRegionProxyActorFactory; +import org.eclipse.ditto.internal.utils.cluster.config.DefaultClusterConfig; +import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; +import org.eclipse.ditto.policies.enforcement.config.DefaultEnforcementConfig; +import org.eclipse.ditto.policies.enforcement.config.EnforcementConfig; + +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; + +/** + * checks for the existence of connections. + */ +final class ConnectionExistenceChecker { + + public static final String ENFORCEMENT_CACHE_DISPATCHER = "enforcement-cache-dispatcher"; + + private final AsyncCacheLoader> connectionIdLoader; + private final ActorSystem actorSystem; + + ConnectionExistenceChecker(final ActorSystem actorSystem) { + this.actorSystem = actorSystem; + final var enforcementConfig = DefaultEnforcementConfig.of( + DefaultScopedConfig.dittoScoped(actorSystem.settings().config())); + connectionIdLoader = getConnectionIdLoader(actorSystem, enforcementConfig); + } + + private AsyncCacheLoader> getConnectionIdLoader( + final ActorSystem actorSystem, + final EnforcementConfig enforcementConfig) { + + final var clusterConfig = DefaultClusterConfig.of(actorSystem.settings().config().getConfig("ditto.cluster")); + final ShardRegionProxyActorFactory shardRegionProxyActorFactory = + ShardRegionProxyActorFactory.newInstance(actorSystem, clusterConfig); + + final ActorRef connectionShardRegion = shardRegionProxyActorFactory.getShardRegionProxyActor( + ConnectivityMessagingConstants.CLUSTER_ROLE, ConnectivityMessagingConstants.SHARD_REGION); + return new PreEnforcementConnectionIdCacheLoader(enforcementConfig.getAskWithRetryConfig(), + actorSystem.getScheduler(), + connectionShardRegion); + } + + public CompletionStage checkExistence(final ModifyConnection signal) { + try { + return connectionIdLoader.asyncLoad(signal.getEntityId(), + actorSystem.dispatchers().lookup(ENFORCEMENT_CACHE_DISPATCHER)) + .thenApply(Entry::exists); + } catch (final Exception e) { + throw new IllegalStateException("Could not load connection via connectionIdLoader", e); + } + } + +} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ModifyToCreateConnectionTransformer.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ModifyToCreateConnectionTransformer.java new file mode 100644 index 00000000000..5d713fe5781 --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/ModifyToCreateConnectionTransformer.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.service.enforcement.pre; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +import org.eclipse.ditto.base.model.signals.Signal; +import org.eclipse.ditto.base.service.signaltransformer.SignalTransformer; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnection; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; + +import com.typesafe.config.Config; + +import akka.actor.ActorSystem; + +/** + * Transforms a ModifyConnection into a CreateConnection if the connection does not exist already. + */ +public final class ModifyToCreateConnectionTransformer implements SignalTransformer { + + private final ConnectionExistenceChecker existenceChecker; + + @SuppressWarnings("unused") + ModifyToCreateConnectionTransformer(final ActorSystem actorSystem, final Config config) { + this(new ConnectionExistenceChecker(actorSystem)); + } + + ModifyToCreateConnectionTransformer(final ConnectionExistenceChecker existenceChecker) { + this.existenceChecker = existenceChecker; + } + + @Override + public CompletionStage> apply(final Signal signal) { + if (signal instanceof ModifyConnection modifyConnection) { + return existenceChecker.checkExistence(modifyConnection) + .thenApply(exists -> { + if (Boolean.FALSE.equals(exists)) { + return CreateConnection.of( + modifyConnection.getConnection(), + modifyConnection.getDittoHeaders() + ); + } else { + return modifyConnection; + } + }); + } else { + return CompletableFuture.completedStage(signal); + } + } + +} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/PreEnforcementConnectionIdCacheLoader.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/PreEnforcementConnectionIdCacheLoader.java new file mode 100644 index 00000000000..8968c77b109 --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/PreEnforcementConnectionIdCacheLoader.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.service.enforcement.pre; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; + +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.signals.commands.Command; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.ConnectivityConstants; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; +import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection; +import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionResponse; +import org.eclipse.ditto.internal.utils.cache.entry.Entry; +import org.eclipse.ditto.internal.utils.cacheloaders.ActorAskCacheLoader; +import org.eclipse.ditto.internal.utils.cacheloaders.config.AskWithRetryConfig; +import org.eclipse.ditto.json.JsonFieldSelector; + +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; + +import akka.actor.ActorRef; +import akka.actor.Scheduler; + +/** + * Cache loader used for Connection existence check in pre-enforcement. + */ +final class PreEnforcementConnectionIdCacheLoader implements + AsyncCacheLoader> { + + private final ActorAskCacheLoader, ConnectionId> delegate; + + /** + * Constructor. + * + * @param askWithRetryConfig the configuration for the "ask with retry" pattern applied for the cache loader. + * @param scheduler the scheduler to use for the "ask with retry" for retries. + * @param shardRegionProxy the shard-region-proxy. + */ + public PreEnforcementConnectionIdCacheLoader(final AskWithRetryConfig askWithRetryConfig, + final Scheduler scheduler, + final ActorRef shardRegionProxy) { + + delegate = ActorAskCacheLoader.forShard(askWithRetryConfig, + scheduler, + ConnectivityConstants.ENTITY_TYPE, + shardRegionProxy, + connectionId -> RetrieveConnection.of(connectionId, JsonFieldSelector.newInstance("id"), DittoHeaders.empty()), + PreEnforcementConnectionIdCacheLoader::handleRetrieveConnectionResponse); + } + + @Override + public CompletableFuture> asyncLoad(final ConnectionId key, final Executor executor) { + return delegate.asyncLoad(key, executor); + } + + private static Entry handleRetrieveConnectionResponse(final Object response) { + + if (response instanceof RetrieveConnectionResponse retrieveConnectionResponse) { + final ConnectionId connectionId = retrieveConnectionResponse.getEntityId(); + return Entry.of(-1, connectionId); + } else if (response instanceof ConnectionNotAccessibleException) { + return Entry.nonexistent(); + } else { + throw new IllegalStateException("expect RetrieveConnectionResponse, got: " + response); + } + } + +} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/package-info.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/package-info.java new file mode 100644 index 00000000000..35e7d14ecc6 --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/enforcement/pre/package-info.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.eclipse.ditto.utils.jsr305.annotations.AllParametersAndReturnValuesAreNonnullByDefault +package org.eclipse.ditto.connectivity.service.enforcement.pre; diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/stages/StagedCommand.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/stages/StagedCommand.java index 96777760a5a..140dbb0501f 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/stages/StagedCommand.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/stages/StagedCommand.java @@ -27,6 +27,8 @@ import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.connectivity.model.ConnectionId; +import org.eclipse.ditto.connectivity.model.WithConnectionId; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.json.JsonField; @@ -41,7 +43,8 @@ * It contains a sequence of actions. Some actions are asynchronous. The connection actor can thus schedule the next * action as a staged command to self after an asynchronous action. Synchronous actions can be executed right away. */ -public final class StagedCommand implements ConnectivityCommand, Iterator { +public final class StagedCommand implements ConnectivityCommand, Iterator, + WithConnectionId { private final ConnectivityCommand command; @Nullable private final ConnectivityEvent event; @@ -82,6 +85,17 @@ public ConnectivityCommand getCommand() { return command; } + @Override + public ConnectionId getEntityId() { + if (command instanceof WithConnectionId withConnectionId) { + return withConnectionId.getEntityId(); + } else if (event != null) { + return event.getEntityId(); + } else { + throw new IllegalStateException("Could not determine ConnectionId in StagedCommand"); + } + } + /** * @return the event to persist, apply or publish or dummy-event. */ @@ -210,4 +224,5 @@ public ConnectionAction nextAction() { private Queue getActionsAsQueue() { return new LinkedList<>(actions); } + } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/AbstractConnectivityCommandStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/AbstractConnectivityCommandStrategy.java index 2d11f67ebbf..65608619c5e 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/AbstractConnectivityCommandStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/AbstractConnectivityCommandStrategy.java @@ -29,7 +29,8 @@ import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.commands.AbstractCommandStrategy; +import org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator; +import org.eclipse.ditto.internal.utils.persistentactors.etags.AbstractConditionHeaderCheckingCommandStrategy; /** * Abstract base class for {@link org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityCommand} strategies. @@ -37,12 +38,20 @@ * @param the type of the handled command */ abstract class AbstractConnectivityCommandStrategy> - extends AbstractCommandStrategy> { + extends AbstractConditionHeaderCheckingCommandStrategy> { - AbstractConnectivityCommandStrategy(final Class theMatchingClass) { + private static final ConditionalHeadersValidator VALIDATOR = + ConnectionsConditionalHeadersValidatorProvider.getInstance(); + + protected AbstractConnectivityCommandStrategy(final Class theMatchingClass) { super(theMatchingClass); } + @Override + protected ConditionalHeadersValidator getValidator() { + return VALIDATOR; + } + @Override public boolean isDefined(final C command) { return command instanceof ConnectivityCommand || command instanceof ConnectivitySudoCommand; diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CloseConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CloseConnectionStrategy.java index 3b3bb0e322e..36db15991b5 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CloseConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CloseConnectionStrategy.java @@ -16,20 +16,22 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CloseConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CloseConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionClosed; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.CloseConnection} command. @@ -56,4 +58,15 @@ protected Result> doApply(final Context co ConnectionAction.SEND_RESPONSE); return newMutationResult(StagedCommand.of(command, event, response, actions), event, response); } + + @Override + public Optional previousEntityTag(final CloseConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final CloseConnection command, @Nullable final Connection newEntity) { + return Optional.of(getEntityOrThrow(newEntity)).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionConflictStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionConflictStrategy.java index 8387721afdc..9d6050e93fc 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionConflictStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionConflictStrategy.java @@ -14,15 +14,18 @@ import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newErrorResult; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionConflictException; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnection; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnection} command @@ -49,4 +52,15 @@ protected Result> doApply(final Context co .build(); return newErrorResult(conflictException, command); } + + @Override + public Optional previousEntityTag(final CreateConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final CreateConnection command, @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionUninitializedStrategies.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionUninitializedStrategies.java deleted file mode 100644 index f606c1e0215..00000000000 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionUninitializedStrategies.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2021 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ -package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; - -import java.util.Optional; -import java.util.function.Consumer; - -import javax.annotation.Nullable; - -import org.eclipse.ditto.base.model.entity.metadata.Metadata; -import org.eclipse.ditto.base.model.signals.commands.Command; -import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; -import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.commands.AbstractCommandStrategy; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; -import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; - -/** - * Strategies to handle signals as an uninitialized connection that stashes every command. - */ -public final class ConnectionUninitializedStrategies - extends AbstractCommandStrategy, Connection, ConnectionState, ConnectivityEvent> - implements ConnectivityCommandStrategies { - - private final Consumer> action; - - private ConnectionUninitializedStrategies(final Consumer> action) { - super(Command.class); - this.action = action; - } - - /** - * Return a new instance of this class. - * - * @param action what to do on connectivity commands. - * @return the empty result. - */ - public static ConnectionUninitializedStrategies of(final Consumer> action) { - return new ConnectionUninitializedStrategies(action); - } - - @Override - public boolean isDefined(final Command command) { - return true; - } - - @Override - protected Optional calculateRelativeMetadata(@Nullable final Connection entity, - final Command command) { - return Optional.empty(); - } - - @Override - protected Result> doApply(final Context context, - @Nullable final Connection entity, - final long nextRevision, final Command command, @Nullable final Metadata metadata) { - action.accept(command); - return Result.empty(); - } - - @Override - public Result> unhandled(final Context context, - @Nullable final Connection entity, - final long nextRevision, - final Command command) { - - return ResultFactory.newErrorResult(ConnectionNotAccessibleException - .newBuilder(context.getState().id()) - .dittoHeaders(command.getDittoHeaders()) - .build(), command); - } - -} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionsConditionalHeadersValidatorProvider.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionsConditionalHeadersValidatorProvider.java new file mode 100644 index 00000000000..d83645f573e --- /dev/null +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ConnectionsConditionalHeadersValidatorProvider.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; + +import javax.annotation.concurrent.Immutable; + +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeExceptionBuilder; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionPreconditionFailedException; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionPreconditionNotModifiedException; +import org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator; + +/** + * Provides a {@link ConditionalHeadersValidator} which checks conditional (http) headers based on a given ETag on + * Connection resources. + */ +@Immutable +final class ConnectionsConditionalHeadersValidatorProvider { + + /** + * Settings for validating conditional headers on Connection resources. + */ + private static class ConnectionsConditionalHeadersValidationSettings + implements ConditionalHeadersValidator.ValidationSettings { + + /** + * Returns a builder for a {@link org.eclipse.ditto.things.model.signals.commands.exceptions.ThingPreconditionFailedException}. + * + * @param conditionalHeaderName the name of the conditional header. + * @param expected the expected value. + * @param actual the actual ETag value. + * @return the builder. + */ + @Override + public DittoRuntimeExceptionBuilder createPreconditionFailedExceptionBuilder(final String conditionalHeaderName, + final String expected, final String actual) { + return ConnectionPreconditionFailedException.newBuilder(conditionalHeaderName, expected, actual); + } + + /** + * Returns a builder for a {@link ConnectionPreconditionNotModifiedException}. + * + * @param expectedNotToMatch the value which was expected not to match {@code matched} value. + * @param matched the matched value. + * @return the builder. + */ + @Override + public DittoRuntimeExceptionBuilder createPreconditionNotModifiedExceptionBuilder( + final String expectedNotToMatch, final String matched) { + return ConnectionPreconditionNotModifiedException.newBuilder(expectedNotToMatch, matched); + } + } + + private static final ConditionalHeadersValidator INSTANCE = createInstance(); + + private ConnectionsConditionalHeadersValidatorProvider() { + throw new AssertionError(); + } + + /** + * Returns the (singleton) instance of {@link org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator} for Thing resources. + * + * @return the {@link org.eclipse.ditto.internal.utils.headers.conditional.ConditionalHeadersValidator}. + */ + public static ConditionalHeadersValidator getInstance() { + return INSTANCE; + } + + private static ConditionalHeadersValidator createInstance() { + return ConditionalHeadersValidator.of(new ConnectionsConditionalHeadersValidationSettings(), + checker -> false); + } + +} diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CreateConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CreateConnectionStrategy.java index e4978b5a5e0..0e640ab3cc0 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CreateConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/CreateConnectionStrategy.java @@ -22,7 +22,9 @@ import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newErrorResult; import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newMutationResult; +import java.time.Instant; import java.util.List; +import java.util.Objects; import java.util.Optional; import javax.annotation.Nullable; @@ -30,16 +32,17 @@ import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityStatus; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.CreateConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link CreateConnection} command. @@ -50,6 +53,21 @@ final class CreateConnectionStrategy extends AbstractConnectivityCommandStrategy super(CreateConnection.class); } + @Override + public boolean isDefined(final CreateConnection command) { + return true; + } + + @Override + public boolean isDefined(final Context context, @Nullable final Connection connection, + final CreateConnection command) { + final boolean connectionExists = Optional.ofNullable(connection) + .map(t -> !t.isDeleted()) + .orElse(false); + + return !connectionExists && Objects.equals(context.getState().id(), command.getEntityId()); + } + @Override protected Result> doApply(final Context context, @Nullable final Connection entity, @@ -57,7 +75,12 @@ protected Result> doApply(final Context co final CreateConnection command, @Nullable final Metadata metadata) { - final Connection connection = command.getConnection().toBuilder().lifecycle(ACTIVE).build(); + final Instant timestamp = getEventTimestamp(); + final Connection connection = command.getConnection().toBuilder().lifecycle(ACTIVE) + .revision(nextRevision) + .created(timestamp) + .modified(timestamp) + .build(); final ConnectivityEvent event = ConnectionCreated.of(connection, nextRevision, getEventTimestamp(), command.getDittoHeaders(), metadata); @@ -78,4 +101,15 @@ protected Result> doApply(final Context co return newMutationResult(command, event, response, true, false); } } + + @Override + public Optional previousEntityTag(final CreateConnection command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final CreateConnection command, @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/DeleteConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/DeleteConnectionStrategy.java index f27f0559c68..459aed74e2d 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/DeleteConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/DeleteConnectionStrategy.java @@ -16,23 +16,25 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.DeleteConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.DeleteConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionDeleted; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.DeleteConnection} + * This strategy handles the {@link DeleteConnection} * command. */ final class DeleteConnectionStrategy extends AbstractConnectivityCommandStrategy { @@ -61,4 +63,14 @@ protected Result> doApply(final Context co return newMutationResult(StagedCommand.of(command, event, response, actions), event, response); } + @Override + public Optional previousEntityTag(final DeleteConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final DeleteConnection command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/EnableConnectionLogsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/EnableConnectionLogsStrategy.java index b6047eaebbd..5f2c17b9fe2 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/EnableConnectionLogsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/EnableConnectionLogsStrategy.java @@ -14,16 +14,21 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; + +import javax.annotation.Nullable; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.EnableConnectionLogs; import org.eclipse.ditto.connectivity.model.signals.commands.modify.EnableConnectionLogsResponse; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.EnableConnectionLogs} command. + * This strategy handles the {@link EnableConnectionLogs} command. */ final class EnableConnectionLogsStrategy extends AbstractEphemeralStrategy { @@ -40,4 +45,15 @@ WithDittoHeaders getResponse(final ConnectionState state, final DittoHeaders hea List getActions() { return Arrays.asList(ConnectionAction.BROADCAST_TO_CLIENT_ACTORS_IF_STARTED, ConnectionAction.SEND_RESPONSE, ConnectionAction.ENABLE_LOGGING); } + + @Override + public Optional previousEntityTag(final EnableConnectionLogs command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final EnableConnectionLogs command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/LoggingExpiredStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/LoggingExpiredStrategy.java index 81f09893640..e44ccc0b1f5 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/LoggingExpiredStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/LoggingExpiredStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import java.util.Optional; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.LoggingExpired; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.LoggingExpired} command. + * This strategy handles the {@link LoggingExpired} command. */ final class LoggingExpiredStrategy extends AbstractSingleActionStrategy { @@ -28,4 +34,15 @@ final class LoggingExpiredStrategy extends AbstractSingleActionStrategy previousEntityTag(final LoggingExpired command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final LoggingExpired command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ModifyConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ModifyConnectionStrategy.java index d8252b31f1b..2c9813c8a49 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ModifyConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ModifyConnectionStrategy.java @@ -16,6 +16,7 @@ import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newErrorResult; import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newMutationResult; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -25,21 +26,22 @@ import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionConfigurationInvalidException; import org.eclipse.ditto.connectivity.model.ConnectivityStatus; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; +import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnectionResponse; +import org.eclipse.ditto.connectivity.model.signals.events.ConnectionModified; +import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; -import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; -import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnectionResponse; -import org.eclipse.ditto.connectivity.model.signals.events.ConnectionModified; -import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection} command. + * This strategy handles the {@link ModifyConnection} command. */ final class ModifyConnectionStrategy extends AbstractConnectivityCommandStrategy { @@ -54,7 +56,11 @@ protected Result> doApply(final Context co final ModifyConnection command, @Nullable final Metadata metadata) { - final Connection connection = command.getConnection().toBuilder().lifecycle(ACTIVE).build(); + final Instant eventTs = getEventTimestamp(); + final Connection connection = command.getConnection().toBuilder().lifecycle(ACTIVE) + .revision(nextRevision) + .modified(eventTs) + .build(); if (entity != null && entity.getConnectionType() != connection.getConnectionType() && !command.getDittoHeaders().isSudo()) { return ResultFactory.newErrorResult( @@ -96,4 +102,15 @@ protected Result> doApply(final Context co return newMutationResult(command, event, response); } } + + @Override + public Optional previousEntityTag(final ModifyConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final ModifyConnection command, @Nullable final Connection newEntity) { + return Optional.of(getEntityOrThrow(newEntity)).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/OpenConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/OpenConnectionStrategy.java index f5978d4ec57..14048528575 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/OpenConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/OpenConnectionStrategy.java @@ -29,15 +29,16 @@ import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.OpenConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.OpenConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionOpened; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link OpenConnection} command. @@ -69,4 +70,15 @@ protected Result> doApply(final Context co return newMutationResult(StagedCommand.of(command, event, response, actions), event, response); } } + + @Override + public Optional previousEntityTag(final OpenConnection command, + @Nullable final Connection previousEntity) { + return Optional.ofNullable(previousEntity).flatMap(EntityTag::fromEntity); + } + + @Override + public Optional nextEntityTag(final OpenConnection command, @Nullable final Connection newEntity) { + return Optional.of(getEntityOrThrow(newEntity)).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionLogsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionLogsStrategy.java index aa80d590d8a..fc59fe58805 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionLogsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionLogsStrategy.java @@ -14,13 +14,18 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; + +import javax.annotation.Nullable; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionLogs; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionLogsResponse; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionLogs} @@ -41,4 +46,15 @@ WithDittoHeaders getResponse(final ConnectionState state, final DittoHeaders hea List getActions() { return Arrays.asList(ConnectionAction.BROADCAST_TO_CLIENT_ACTORS_IF_STARTED, ConnectionAction.SEND_RESPONSE); } + + @Override + public Optional previousEntityTag(final ResetConnectionLogs command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final ResetConnectionLogs command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionMetricsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionMetricsStrategy.java index 40db31d9de2..f7a3faf7625 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionMetricsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/ResetConnectionMetricsStrategy.java @@ -14,16 +14,21 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; + +import javax.annotation.Nullable; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionMetrics; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionMetricsResponse; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.ResetConnectionMetrics} + * This strategy handles the {@link ResetConnectionMetrics} * command. */ final class ResetConnectionMetricsStrategy extends AbstractEphemeralStrategy { @@ -41,4 +46,16 @@ WithDittoHeaders getResponse(final ConnectionState state, final DittoHeaders hea List getActions() { return Arrays.asList(ConnectionAction.BROADCAST_TO_CLIENT_ACTORS_IF_STARTED, ConnectionAction.SEND_RESPONSE); } + + @Override + public Optional previousEntityTag(final ResetConnectionMetrics command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final ResetConnectionMetrics command, + @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionLogsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionLogsStrategy.java index c93539703cf..988b39526cc 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionLogsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionLogsStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import java.util.Optional; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionLogs; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionLogs} + * This strategy handles the {@link RetrieveConnectionLogs} * command. */ final class RetrieveConnectionLogsStrategy extends AbstractSingleActionStrategy { @@ -29,4 +35,16 @@ final class RetrieveConnectionLogsStrategy extends AbstractSingleActionStrategy< ConnectionAction getAction() { return ConnectionAction.RETRIEVE_CONNECTION_LOGS; } + + @Override + public Optional previousEntityTag(final RetrieveConnectionLogs command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveConnectionLogs command, + @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionMetricsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionMetricsStrategy.java index a6de954cfb6..c9f801253a1 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionMetricsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionMetricsStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import java.util.Optional; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionMetrics; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionMetrics} + * This strategy handles the {@link RetrieveConnectionMetrics} * command. */ final class RetrieveConnectionMetricsStrategy extends AbstractSingleActionStrategy { @@ -29,4 +35,16 @@ final class RetrieveConnectionMetricsStrategy extends AbstractSingleActionStrate ConnectionAction getAction() { return ConnectionAction.RETRIEVE_CONNECTION_METRICS; } + + @Override + public Optional previousEntityTag(final RetrieveConnectionMetrics command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveConnectionMetrics command, + @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStatusStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStatusStrategy.java index 98bba007344..17d63c06cf3 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStatusStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStatusStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import java.util.Optional; + +import javax.annotation.Nullable; + +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; +import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionStatus; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionStatus} + * This strategy handles the {@link RetrieveConnectionStatus} * command. */ final class RetrieveConnectionStatusStrategy extends AbstractSingleActionStrategy { @@ -29,4 +35,16 @@ final class RetrieveConnectionStatusStrategy extends AbstractSingleActionStrateg ConnectionAction getAction() { return ConnectionAction.RETRIEVE_CONNECTION_STATUS; } + + @Override + public Optional previousEntityTag(final RetrieveConnectionStatus command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveConnectionStatus command, + @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStrategy.java index 0efc54c8747..8afe2019b92 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveConnectionStrategy.java @@ -12,19 +12,26 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.DittoHeadersSettable; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; -import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; +import org.eclipse.ditto.connectivity.model.signals.commands.query.ConnectivityQueryCommand; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; +import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; +import org.eclipse.ditto.json.JsonObject; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection} command. + * This strategy handles the {@link RetrieveConnection} command. */ final class RetrieveConnectionStrategy extends AbstractConnectivityCommandStrategy { @@ -41,9 +48,40 @@ protected Result> doApply(final Context co if (entity != null) { return ResultFactory.newQueryResult(command, - RetrieveConnectionResponse.of(entity.toJson(), command.getDittoHeaders())); + appendETagHeaderIfProvided(command, getRetrieveConnectionResponse(entity, command), entity) + ); } else { return ResultFactory.newErrorResult(notAccessible(context, command), command); } } + + @Override + public Optional previousEntityTag(final RetrieveConnection command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveConnection command, @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } + + private static DittoHeadersSettable getRetrieveConnectionResponse(@Nullable final Connection connection, + final ConnectivityQueryCommand command) { + if (connection != null) { + return RetrieveConnectionResponse.of(getConnectionJson(connection, command), + command.getDittoHeaders()); + } else { + return ConnectionNotAccessibleException.newBuilder(((RetrieveConnection) command).getEntityId()) + .dittoHeaders(command.getDittoHeaders()) + .build(); + } + } + + private static JsonObject getConnectionJson(final Connection connection, + final ConnectivityQueryCommand command) { + return ((RetrieveConnection) command).getSelectedFields() + .map(selectedFields -> connection.toJson(command.getImplementedSchemaVersion(), selectedFields)) + .orElseGet(() -> connection.toJson(command.getImplementedSchemaVersion())); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveResolvedHonoConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveResolvedHonoConnectionStrategy.java index d997d23cb4f..aef990c5d4c 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveResolvedHonoConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/RetrieveResolvedHonoConnectionStrategy.java @@ -12,11 +12,17 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.DittoHeadersSettable; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionType; +import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionNotAccessibleException; +import org.eclipse.ditto.connectivity.model.signals.commands.query.ConnectivityQueryCommand; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveResolvedHonoConnection; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; @@ -24,6 +30,7 @@ import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; +import org.eclipse.ditto.json.JsonObject; import akka.actor.ActorSystem; @@ -49,13 +56,45 @@ protected Result> doApply(final Context co final Result> result; if (entity != null && entity.getConnectionType() == ConnectionType.HONO) { - final var json = honoConnectionFactory.getHonoConnection(entity).toJson(); - - result = ResultFactory.newQueryResult(command, - RetrieveConnectionResponse.of(json, command.getDittoHeaders())); + return ResultFactory.newQueryResult(command, + appendETagHeaderIfProvided(command, getRetrieveConnectionResponse(entity, command), entity) + ); } else { result = ResultFactory.newErrorResult(notAccessible(context, command), command); } return result; } + + @Override + public Optional previousEntityTag(final RetrieveResolvedHonoConnection command, + @Nullable final Connection previousEntity) { + return nextEntityTag(command, previousEntity); + } + + @Override + public Optional nextEntityTag(final RetrieveResolvedHonoConnection command, + @Nullable final Connection newEntity) { + return Optional.ofNullable(newEntity).flatMap(EntityTag::fromEntity); + } + + private DittoHeadersSettable getRetrieveConnectionResponse(@Nullable final Connection connection, + final ConnectivityQueryCommand command) { + if (connection != null) { + return RetrieveConnectionResponse.of(getConnectionJson(connection, command), + command.getDittoHeaders()); + } else { + return ConnectionNotAccessibleException.newBuilder(((RetrieveResolvedHonoConnection) command).getEntityId()) + .dittoHeaders(command.getDittoHeaders()) + .build(); + } + } + + private JsonObject getConnectionJson(final Connection connection, + final ConnectivityQueryCommand command) { + + final Connection honoConnection = honoConnectionFactory.getHonoConnection(connection); + return ((RetrieveResolvedHonoConnection) command).getSelectedFields() + .map(selectedFields -> honoConnection.toJson(command.getImplementedSchemaVersion(), selectedFields)) + .orElseGet(() -> honoConnection.toJson(command.getImplementedSchemaVersion())); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/StagedCommandStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/StagedCommandStrategy.java index b35efef2929..50921897e83 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/StagedCommandStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/StagedCommandStrategy.java @@ -13,17 +13,19 @@ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; import java.text.MessageFormat; +import java.util.Optional; import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityInternalErrorException; +import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory; -import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand} @@ -50,4 +52,15 @@ protected Result> doApply(final Context co .dittoHeaders(command.getDittoHeaders()) .build(), command)); } + + @Override + public Optional previousEntityTag(final StagedCommand command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final StagedCommand command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoAddConnectionLogEntryStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoAddConnectionLogEntryStrategy.java index 9f71bd93e8b..3c59a182321 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoAddConnectionLogEntryStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoAddConnectionLogEntryStrategy.java @@ -12,9 +12,12 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoAddConnectionLogEntry; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionId; @@ -58,4 +61,16 @@ private ConnectionLogger getAppropriateLogger(final ConnectionLoggerRegistry con logEntry.getLogType(), logEntry.getAddress().orElse(null)); } + + @Override + public Optional previousEntityTag(final SudoAddConnectionLogEntry command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final SudoAddConnectionLogEntry command, + @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoRetrieveConnectionTagsStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoRetrieveConnectionTagsStrategy.java index 0b8b7a30fa9..f447b56f18e 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoRetrieveConnectionTagsStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/SudoRetrieveConnectionTagsStrategy.java @@ -12,9 +12,12 @@ */ package org.eclipse.ditto.connectivity.service.messaging.persistence.strategies.commands; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionTags; import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionTagsResponse; import org.eclipse.ditto.connectivity.model.Connection; @@ -46,4 +49,16 @@ protected Result> doApply(final Context co return ResultFactory.newErrorResult(notAccessible(context, command), command); } } + + @Override + public Optional previousEntityTag(final SudoRetrieveConnectionTags command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final SudoRetrieveConnectionTags command, + @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionConflictStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionConflictStrategy.java index 6837d04401d..7ce4217df36 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionConflictStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionConflictStrategy.java @@ -14,15 +14,18 @@ import static org.eclipse.ditto.internal.utils.persistentactors.results.ResultFactory.newQueryResult; +import java.util.Optional; + import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnection} command @@ -44,4 +47,15 @@ protected Result> doApply(final Context co return newQueryResult(command, TestConnectionResponse.alreadyCreated(context.getState().id(), command.getDittoHeaders())); } + + @Override + public Optional previousEntityTag(final TestConnection command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final TestConnection command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionStrategy.java index 42ae1cc3514..4001ceb4f75 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/commands/TestConnectionStrategy.java @@ -18,21 +18,23 @@ import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import javax.annotation.Nullable; import org.eclipse.ditto.base.model.entity.metadata.Metadata; import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; -import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; -import org.eclipse.ditto.internal.utils.persistentactors.results.Result; import org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnection; import org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; import org.eclipse.ditto.connectivity.model.signals.events.ConnectivityEvent; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionAction; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.ConnectionState; +import org.eclipse.ditto.connectivity.service.messaging.persistence.stages.StagedCommand; +import org.eclipse.ditto.internal.utils.persistentactors.results.Result; /** * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnection} command. @@ -43,6 +45,20 @@ final class TestConnectionStrategy extends AbstractConnectivityCommandStrategy context, @Nullable final Connection connection, final TestConnection command) { + final boolean connectionExists = Optional.ofNullable(connection) + .map(t -> !t.isDeleted()) + .orElse(false); + + return !connectionExists && Objects.equals(context.getState().id(), command.getEntityId()); + } + @Override protected Result> doApply(final Context context, @Nullable final Connection entity, @@ -67,4 +83,15 @@ protected Result> doApply(final Context co TestConnectionResponse.alreadyCreated(context.getState().id(), command.getDittoHeaders())); } } + + @Override + public Optional previousEntityTag(final TestConnection command, + @Nullable final Connection previousEntity) { + return Optional.empty(); + } + + @Override + public Optional nextEntityTag(final TestConnection command, @Nullable final Connection newEntity) { + return Optional.empty(); + } } diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionClosedStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionClosedStrategy.java index f106c7b3c4d..cc65c9f2dfc 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionClosedStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionClosedStrategy.java @@ -18,11 +18,11 @@ import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityStatus; -import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionClosed; +import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.events.ConnectionClosed} event. + * This strategy handles the {@link ConnectionClosed} event. */ final class ConnectionClosedStrategy implements EventStrategy { diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionCreatedStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionCreatedStrategy.java index 10fc23649c7..27cabadce17 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionCreatedStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionCreatedStrategy.java @@ -15,11 +15,11 @@ import javax.annotation.Nullable; import org.eclipse.ditto.connectivity.model.Connection; -import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated; +import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.events.ConnectionCreated} event. + * This strategy handles the {@link ConnectionCreated} event. */ final class ConnectionCreatedStrategy implements EventStrategy { diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionDeletedStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionDeletedStrategy.java index ed958633104..5709cda7baf 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionDeletedStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionDeletedStrategy.java @@ -16,11 +16,11 @@ import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionLifecycle; -import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionDeleted; +import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.events.ConnectionDeleted} event. + * This strategy handles the {@link ConnectionDeleted} event. */ final class ConnectionDeletedStrategy implements EventStrategy { @@ -29,7 +29,9 @@ final class ConnectionDeletedStrategy implements EventStrategy { diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionOpenedStrategy.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionOpenedStrategy.java index 517b0986a45..884dbd10742 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionOpenedStrategy.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/persistence/strategies/events/ConnectionOpenedStrategy.java @@ -18,17 +18,20 @@ import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectivityStatus; -import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; import org.eclipse.ditto.connectivity.model.signals.events.ConnectionOpened; +import org.eclipse.ditto.internal.utils.persistentactors.events.EventStrategy; /** - * This strategy handles the {@link org.eclipse.ditto.connectivity.model.signals.events.ConnectionOpened} event. + * This strategy handles the {@link ConnectionOpened} event. */ final class ConnectionOpenedStrategy implements EventStrategy { @Override public Connection handle(final ConnectionOpened event, @Nullable final Connection connection, final long revision) { - return checkNotNull(connection, "connection").toBuilder().connectionStatus(ConnectivityStatus.OPEN).build(); + return checkNotNull(connection, "connection") + .toBuilder() + .connectionStatus(ConnectivityStatus.OPEN) + .build(); } } diff --git a/connectivity/service/src/main/resources/connectivity-extension.conf b/connectivity/service/src/main/resources/connectivity-extension.conf index e69de29bb2d..0529d13a7a2 100644 --- a/connectivity/service/src/main/resources/connectivity-extension.conf +++ b/connectivity/service/src/main/resources/connectivity-extension.conf @@ -0,0 +1,5 @@ +ditto.extensions { + signal-transformers-provider.extension-config.signal-transformers = [ + "org.eclipse.ditto.connectivity.service.enforcement.pre.ModifyToCreateConnectionTransformer", // always keep this as first transformer in order to guarantee that all following transformers know that the command is creating a connection instead of modifying it + ] ${ditto.extensions.signal-transformers-provider.extension-config.signal-transformers} +} diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/ErrorHandlingActorTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/ErrorHandlingActorTest.java index 7391eb710c3..d40dd99cad9 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/ErrorHandlingActorTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/ErrorHandlingActorTest.java @@ -16,6 +16,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import org.assertj.core.api.Assertions; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionId; @@ -91,7 +92,12 @@ public void tryCreateConnectionExpectSuccessResponseIndependentOfConnectionStatu // create connection final ConnectivityModifyCommand command = CreateConnection.of(connection, DittoHeaders.empty()); underTest.tell(command, getRef()); - expectMsg(CreateConnectionResponse.of(connection, DittoHeaders.empty())); + final CreateConnectionResponse resp = + expectMsgClass(dilated(CONNECT_TIMEOUT), CreateConnectionResponse.class); + Assertions.assertThat(resp.getConnection()) + .usingRecursiveComparison() + .ignoringFields("revision", "modified", "created") + .isEqualTo(connection); }}; tearDown(); } @@ -121,9 +127,12 @@ public void tryDeleteConnectionExpectErrorResponse() { // create connection final CreateConnection createConnection = CreateConnection.of(connection, DittoHeaders.empty()); underTest.tell(createConnection, getRef()); - final CreateConnectionResponse createConnectionResponse = - CreateConnectionResponse.of(connection, DittoHeaders.empty()); - expectMsg(dilated(CONNECT_TIMEOUT), createConnectionResponse); + final CreateConnectionResponse resp = + expectMsgClass(dilated(CONNECT_TIMEOUT), CreateConnectionResponse.class); + Assertions.assertThat(resp.getConnection()) + .usingRecursiveComparison() + .ignoringFields("revision", "modified", "created") + .isEqualTo(connection); // delete connection final ConnectivityModifyCommand command = DeleteConnection.of(connectionId, DittoHeaders.empty()); @@ -147,9 +156,12 @@ private void tryModifyConnectionExpectErrorResponse(final String action) { // create connection final CreateConnection createConnection = CreateConnection.of(connection, DittoHeaders.empty()); underTest.tell(createConnection, getRef()); - final CreateConnectionResponse createConnectionResponse = - CreateConnectionResponse.of(connection, DittoHeaders.empty()); - expectMsg(createConnectionResponse); + final CreateConnectionResponse resp = + expectMsgClass(dilated(CONNECT_TIMEOUT), CreateConnectionResponse.class); + Assertions.assertThat(resp.getConnection()) + .usingRecursiveComparison() + .ignoringFields("revision", "modified", "created") + .isEqualTo(connection); // modify connection final ConnectivityModifyCommand command; diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java index 8362c2e16b5..31d1320b877 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.io.InputStreamReader; -import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Collections; @@ -28,12 +27,14 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistenceResponse; import org.eclipse.ditto.base.model.correlationid.TestNameCorrelationId; import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition; import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.entitytag.EntityTag; import org.eclipse.ditto.connectivity.api.BaseClientState; import org.eclipse.ditto.connectivity.model.Connection; import org.eclipse.ditto.connectivity.model.ConnectionConfigurationInvalidException; @@ -212,8 +213,14 @@ public void testConnection() { @Test public void testConnectionTypeHono() throws IOException { //GIVEN - final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json"); - final var expectedHonoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-expected.json"); + final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json") + .toBuilder() + .id(connectionId) + .build(); + final var expectedHonoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-expected.json") + .toBuilder() + .id(connectionId) + .build(); final var testConnection = TestConnection.of(honoConnection, dittoHeadersWithCorrelationId); final var testProbe = actorSystemResource1.newTestProbe(); final var connectionSupervisorActor = createSupervisor(); @@ -257,7 +264,10 @@ public TestActor.AutoPilot run(final ActorRef sender, final Object msg) { ConfigFactory.empty())); final var underTest = actorSystemResource1.newActor(connectionActorProps, connectionId.toString()); - underTest.tell(createConnection(honoConnection), testProbe.ref()); + final CreateConnection createConnection = createConnection( + honoConnection.toBuilder().id(connectionId).build() + ); + underTest.tell(createConnection, testProbe.ref()); testProbe.expectMsgClass(FiniteDuration.apply(20, "s"), CreateConnectionResponse.class); Arrays.stream(ConnectionType.values()).forEach(connectionType -> { @@ -353,7 +363,7 @@ public void manageConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // close connection final CloseConnection closeConnection = CloseConnection.of(connectionId, dittoHeadersWithCorrelationId); @@ -378,7 +388,7 @@ public void deleteConnectionUpdatesSubscriptionsAndClosesConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // delete connection underTest.tell(DeleteConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -468,7 +478,7 @@ public void createConnectionAfterDeleted() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // delete connection underTest.tell(DeleteConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -479,7 +489,7 @@ public void createConnectionAfterDeleted() { // create connection again (while ConnectionActor is in deleted state) underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); } @Test @@ -491,7 +501,7 @@ public void openConnectionAfterDeletedFails() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // delete connection underTest.tell(DeleteConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -512,7 +522,7 @@ public void createConnectionInClosedState() { // create connection underTest.tell(createConnection(closedConnection), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse(closedConnection)); + expectCreateConnectionResponse(testProbe, closedConnection); // assert that client actor is not called for closed connection mockClientActorProbe.expectNoMessage(); @@ -677,7 +687,7 @@ public void modifyConnectionInClosedState() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // close connection underTest.tell(CloseConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -700,7 +710,7 @@ public void retrieveMetricsInClosedStateDoesNotStartClientActor() { // create connection underTest.tell(createConnection(closedConnection), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse(closedConnection)); + expectCreateConnectionResponse(testProbe, closedConnection); mockClientActorProbe.expectNoMessage(); // retrieve metrics @@ -724,7 +734,7 @@ public void modifyConnectionClosesAndRestartsClientActor() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // modify connection | Implicitly validates the restart by waiting for pubsub subscribe from client actor. underTest.tell(ModifyConnection.of(connection, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -749,7 +759,7 @@ public void recoverOpenConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // stop actor actorSystemResource1.stopActor(underTest); @@ -800,7 +810,7 @@ public void recoverModifiedConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // modify connection final var modifiedConnection = ConnectivityModelFactory.newConnectionBuilder(connection) @@ -822,7 +832,11 @@ public void recoverModifiedConnection() { // retrieve connection status underTest.tell(RetrieveConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); - testProbe.expectMsg(RetrieveConnectionResponse.of(modifiedConnection.toJson(), dittoHeadersWithCorrelationId)); + testProbe.expectMsg(RetrieveConnectionResponse.of(modifiedConnection.toJson(), + dittoHeadersWithCorrelationId.toBuilder() + .eTag(EntityTag.fromString("\"rev:2\"")) + .build() + )); } @Test @@ -834,7 +848,7 @@ public void recoverClosedConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // close connection underTest.tell(CloseConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -876,7 +890,7 @@ public void recoverDeletedConnection() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // delete connection underTest.tell(DeleteConnection.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -900,7 +914,7 @@ public void exceptionDuringClientActorPropsCreation() { final var supervisor = actorSystemResource1.newTestProbe(); final var connectionActorProps = ConnectionPersistenceActor.props( - TestConstants.createRandomConnectionId(), + connectionId, Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), @@ -925,7 +939,7 @@ public void exceptionDuringClientActorPropsCreation() { @Test public void exceptionDueToCustomValidator() { - final var connectionActorProps = ConnectionPersistenceActor.props(TestConstants.createRandomConnectionId(), + final var connectionActorProps = ConnectionPersistenceActor.props(connectionId, Mockito.mock(MongoReadJournal.class), commandForwarderActor, pubSubMediatorProbe.ref(), @@ -960,7 +974,7 @@ public void testResetConnectionMetrics() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // reset metrics final var resetConnectionMetrics = ResetConnectionMetrics.of(connectionId, dittoHeadersWithCorrelationId); @@ -979,7 +993,7 @@ public void testConnectionActorRespondsToCleanupCommand() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // send cleanup command underTest.tell(CleanupPersistence.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -995,7 +1009,7 @@ public void enableConnectionLogs() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); //Close logging which are automatically enabled via create connection underTest.tell(LoggingExpired.of(connectionId, dittoHeadersWithCorrelationId), testProbe.ref()); @@ -1017,7 +1031,7 @@ public void retrieveLogsInClosedStateDoesNotStartClientActor() { // create connection underTest.tell(createConnection(closedConnection), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse(closedConnection)); + expectCreateConnectionResponse(testProbe, closedConnection); clientActorProbe.expectNoMessage(); // retrieve logs @@ -1048,7 +1062,7 @@ public void retrieveLogsIsAggregated() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // retrieve logs final var retrieveConnectionLogs = RetrieveConnectionLogs.of(connectionId, dittoHeadersWithCorrelationId); @@ -1075,7 +1089,7 @@ public void resetConnectionLogs() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // reset logs underTest.tell(resetConnectionLogs, testProbe.ref()); @@ -1092,7 +1106,7 @@ public void enabledConnectionLogsAreEnabledAgainAfterModify() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); // Wait until connection is established // enable connection logs @@ -1121,7 +1135,7 @@ public void disabledConnectionLogsAreNotEnabledAfterModify() { // create connection underTest.tell(createConnection(), testProbe.ref()); simulateSuccessfulOpenConnectionInClientActor(); - testProbe.expectMsg(createConnectionResponse()); + expectCreateConnectionResponse(testProbe, connection); //Close logging which are automatically enabled via create connection underTest.tell(LoggingExpired.of(connectionId, DittoHeaders.empty()), testProbe.ref()); @@ -1238,9 +1252,13 @@ public void retriesStartingClientActor() { final DittoHeaders headersIndicatingFailingInstantiation = createConnection.getDittoHeaders().toBuilder() .putHeader("number-of-instantiation-failures", String.valueOf(TestConstants.CONNECTION_CONFIG.getClientActorRestartsBeforeEscalation())) + .responseRequired(false) .build(); underTest.tell(createConnection.setDittoHeaders(headersIndicatingFailingInstantiation), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse().setDittoHeaders(headersIndicatingFailingInstantiation)); + final CreateConnectionResponse resp = + expectCreateConnectionResponse(testProbe, connection); + assertThat(resp.getDittoHeaders()) + .isEqualTo(headersIndicatingFailingInstantiation); assertThat(underTest.isTerminated()).isFalse(); } @@ -1266,9 +1284,13 @@ public void escalatesWhenClientActorFailsTooOften() { final DittoHeaders headersIndicatingFailingInstantiation = createConnection.getDittoHeaders().toBuilder() .putHeader("number-of-instantiation-failures", String.valueOf(TestConstants.CONNECTION_CONFIG.getClientActorRestartsBeforeEscalation() + 1)) + .responseRequired(false) .build(); underTest.tell(createConnection.setDittoHeaders(headersIndicatingFailingInstantiation), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse().setDittoHeaders(headersIndicatingFailingInstantiation)); + final CreateConnectionResponse resp = + expectCreateConnectionResponse(testProbe, connection); + assertThat(resp.getDittoHeaders()) + .isEqualTo(headersIndicatingFailingInstantiation); testProbe.expectTerminated(underTest, FiniteDuration.apply(3, TimeUnit.SECONDS)); } @@ -1281,7 +1303,7 @@ public void deleteConnectionCommandEmitsEvent() { // create connection underTest.tell(createConnection(closedConnection), testProbe.ref()); - testProbe.expectMsg(createConnectionResponse(closedConnection)); + expectCreateConnectionResponse(testProbe, closedConnection); pubSubMediatorProbe.expectMsgClass(DistributedPubSubMediator.Subscribe.class); // delete connection @@ -1310,14 +1332,17 @@ private CreateConnection createConnection(final Connection connection) { return CreateConnection.of(connection, dittoHeadersWithCorrelationId); } - private CreateConnectionResponse createConnectionResponse() { - return createConnectionResponse(connection); - } - - private CreateConnectionResponse createConnectionResponse(final Connection connection) { - return CreateConnectionResponse.of(connection, dittoHeadersWithCorrelationId); + private CreateConnectionResponse expectCreateConnectionResponse(final TestProbe probe, + final Connection theConnection) { + final CreateConnectionResponse resp = + probe.expectMsgClass(CreateConnectionResponse.class); + Assertions.assertThat(resp.getConnection()) + .usingRecursiveComparison() + .ignoringFields("revision", "modified", "created") + .isEqualTo(theConnection); + return resp; } - + private void simulateSuccessfulOpenConnectionInClientActor() { expectMockClientActorMessage(EnableConnectionLogs.of(connectionId, DittoHeaders.empty())); expectMockClientActorMessage(OpenConnection.of(connectionId, dittoHeadersWithCorrelationId)); diff --git a/documentation/src/main/resources/jsonschema/connection.json b/documentation/src/main/resources/jsonschema/connection.json index c0c1be174db..ce9f1c7642c 100644 --- a/documentation/src/main/resources/jsonschema/connection.json +++ b/documentation/src/main/resources/jsonschema/connection.json @@ -840,6 +840,20 @@ } } } + }, + "_revision": { + "type": "integer", + "description": "_(read-only)_ The revision is a counter which is incremented on each modification of a connection." + }, + "_created": { + "type": "string", + "description": "_(read-only)_ The created timestamp of the connection in ISO-8601 UTC format. The timestamp is set on creation of a connection.", + "format": "date-time" + }, + "_modified": { + "type": "string", + "description": "_(read-only)_ The modified timestamp of the connection in ISO-8601 UTC format. The timestamp is set on each modification of a connection.", + "format": "date-time" } }, "additionalProperties": false, diff --git a/documentation/src/main/resources/jsonschema/policy.json b/documentation/src/main/resources/jsonschema/policy.json index 71ea2179a8a..cd627629c3a 100644 --- a/documentation/src/main/resources/jsonschema/policy.json +++ b/documentation/src/main/resources/jsonschema/policy.json @@ -1,12 +1,36 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "description": "A Policy enables developers to configure fine-grained access control for Things.", + "description": "A policy enables developers to configure fine-grained access control for Things.", "title": "Policy", "properties": { "policyId": { "type": "string", - "description": "Unique identifier representing the Policy, has to conform to the namespaced entity ID notation (see [Ditto documentation on namespaced entity IDs](https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id)).\n\nExamples for a valid Policy ID:\n * `org.eclipse.ditto:xdk_policy_53`\n * `foo:xdk_53`\n * `org.eclipse.vorto_42:xdk_policy`" + "description": "Unique identifier representing the policy, has to conform to the namespaced entity ID notation (see [Ditto documentation on namespaced entity IDs](https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id)).\n\nExamples for a valid policy ID:\n * `org.eclipse.ditto:xdk_policy_53`\n * `foo:xdk_53`\n * `org.eclipse.vorto_42:xdk_policy`" + }, + "imports": { + "title": "PolicyImports", + "type": "object", + "description": "Policy imports containing one policy import for each key. The key is the policy ID of the referenced policy.", + "properties": { + "additionalProperties": { + "title": "PolicyImport", + "type": "object", + "description": "Single policy import defining which policy entries of the referenced policy are imported.", + "properties": { + "entries" : { + "title": "ImportedEntries", + "type": "array", + "description": "The policy entries to import from the referenced policy identified by their labels. In case the field is omitted or an empty array is provided, all policy entries defined as implicit (\"importable\": \"implicit\") are imported.", + "items": { + "type": "string", + "description": "Label of a policy entry to import from the referenced policy." + }, + "maxItems": 10 + } + } + } + } }, "entries": { "title": "PolicyEntries", @@ -16,7 +40,7 @@ "additionalProperties": { "title": "Label", "type": "object", - "description": "Single Policy entry containing Subjects and Resources.", + "description": "Single policy entry containing Subjects and Resources.", "properties": { "subjects": { "title": "Subjects", @@ -34,7 +58,7 @@ }, "expiry": { "type": "string", - "description": "The optional expiry timestamp (formatted in ISO-8601) indicates how long this subject should be considered before it is automatically deleted from the Policy.", + "description": "The optional expiry timestamp (formatted in ISO-8601) indicates how long this subject should be considered before it is automatically deleted from the policy.", "format": "date-time" }, "requestedAcks": { @@ -70,13 +94,13 @@ "additionalProperties": { "title": "ResourceEntry", "type": "object", - "description": "Single Resource entry defining permissions per effect. The keys must be in the format `type:path` with `type` being one of the following `thing`, `policy` or `message` resources. See [Policy documentation](../basic-policy.html#which-resources-can-be-controlled) for detailed information.", + "description": "Single Resource entry defining permissions per effect. The keys must be in the format `type:path` with `type` being one of the following `thing`, `policy` or `message` resources. See [policy documentation](../basic-policy.html#which-resources-can-be-controlled) for detailed information.", "properties": { "grant": { "type": "array", "items": { "type": "string", - "description": "All subjects specified in this Policy entry are granted read/write permission on the resources specified in the path, and all subsequent paths, except they are revoked at a subsequent policy label.", + "description": "All subjects specified in this policy entry are granted read/write permission on the resources specified in the path, and all subsequent paths, except they are revoked at a subsequent policy label.", "enum": [ "READ", "WRITE" @@ -87,7 +111,7 @@ "type": "array", "items": { "type": "string", - "description": "All subjects specified in this Policy entry are prohibited to read/write on the resources specified in the path, and all subsequent paths, except they are granted again such permission at a subsequent policy label.", + "description": "All subjects specified in this policy entry are prohibited to read/write on the resources specified in the path, and all subsequent paths, except they are granted again such permission at a subsequent policy label.", "enum": [ "READ", "WRITE" @@ -116,29 +140,23 @@ } } }, - "imports": { - "title": "PolicyImports", + "_revision": { + "type": "integer", + "description": "_(read-only)_ The revision is a counter which is incremented on each modification of a policy." + }, + "_created": { + "type": "string", + "description": "_(read-only)_ The created timestamp of the policy in ISO-8601 UTC format. The timestamp is set on creation of a policy.", + "format": "date-time" + }, + "_modified": { + "type": "string", + "description": "_(read-only)_ The modified timestamp of the policy in ISO-8601 UTC format. The timestamp is set on each modification of a policy.", + "format": "date-time" + }, + "_metadata": { "type": "object", - "description": "Policy imports containing one policy import for each key. The key is the policy ID of the referenced policy.", - "properties": { - "additionalProperties": { - "title": "PolicyImport", - "type": "object", - "description": "Single policy import defining which policy entries of the referenced policy are imported.", - "properties": { - "entries" : { - "title": "ImportedEntries", - "type": "array", - "description": "The policy entries to import from the referenced policy identified by their labels. In case the field is omitted or an empty array is provided, all policy entries defined as implicit (\"importable\": \"implicit\") are imported.", - "items": { - "type": "string", - "description": "Label of a policy entry to import from the referenced policy." - }, - "maxItems": 10 - } - } - } - } + "description": "_(read-only)_ The metadata of the policy. This field is not returned by default but must be selected explicitly. The content is a JSON object having the policy's JSON structure with the difference that the JSON leaves of the policy are JSON objects containing the metadata." } }, "required": [ diff --git a/documentation/src/main/resources/jsonschema/thing_v2.json b/documentation/src/main/resources/jsonschema/thing_v2.json index c17f67ada98..d7ef22efbdf 100644 --- a/documentation/src/main/resources/jsonschema/thing_v2.json +++ b/documentation/src/main/resources/jsonschema/thing_v2.json @@ -1,36 +1,36 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "description": "A *Thing* is a generic entity which can be used as a handle for multiple *Features* belonging to this *Thing*.", + "description": "A *thing* is a generic entity which can be used as a handle for multiple *Features* belonging to this *thing*.", "title": "Thing in API v2", "properties": { "thingId": { "type": "string", - "description": "Unique identifier representing the Thing, has to conform to the namespaced entity ID notation (see [Ditto documentation on namespaced entity IDs](https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id)).\n\nExamples for a valid Thing ID:\n * `org.eclipse.ditto:xdk_53`\n * `foo:xdk_53`\n * `org.eclipse.vorto_42:xdk_thing`" + "description": "Unique identifier representing the thing, has to conform to the namespaced entity ID notation (see [Ditto documentation on namespaced entity IDs](https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id)).\n\nExamples for a valid thing ID:\n * `org.eclipse.ditto:xdk_53`\n * `foo:xdk_53`\n * `org.eclipse.vorto_42:xdk_thing`" }, "policyId": { "type": "string", - "description": "Links to the ID of an existing Policy which contains the authorization information applied for this Thing. The policy ID has to conform to the namespaced entity ID notation (see [Ditto documentation on namespaced entity IDs](https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id))." + "description": "Links to the ID of an existing Policy which contains the authorization information applied for this thing. The policy ID has to conform to the namespaced entity ID notation (see [Ditto documentation on namespaced entity IDs](https://www.eclipse.org/ditto/basic-namespaces-and-names.html#namespaced-id))." }, "definition": { "title": "Definition", "type": "string", - "description": "The definition of this Thing declaring its model in the form '::' or a valid HTTP(s) URL.", + "description": "The definition of this thing declaring its model in the form '::' or a valid HTTP(s) URL.", "pattern": "([_a-zA-Z0-9\\-.]+):([_a-zA-Z0-9\\-.]+):([_a-zA-Z0-9\\-.]+)" }, "attributes": { "title": "Attributes", "type": "object", - "description": "The Attributes that describe this Thing in more detail. Can be an arbitrary JSON object. Attributes are typically used to model rather static properties at the Thing level. Static means that the values do not change as frequently as property values of Features." + "description": "The Attributes that describe this thing in more detail. Can be an arbitrary JSON object. Attributes are typically used to model rather static properties at the thing level. Static means that the values do not change as frequently as property values of Features." }, "features": { "title": "Features", "type": "object", - "description": "The Features belonging to this Thing. A Thing can handle any number of Features.\n The key of this object represents the `featureId`. Due to the fact that a Feature ID often needs to be set in the path of a HTTP request, we strongly recommend to use a restricted the set of characters (e.g. those for [Uniform Resource Identifiers (URI)](http://www.ietf.org/rfc/rfc3986.txt)).", + "description": "The Features belonging to this thing. A thing can handle any number of Features.\n The key of this object represents the `featureId`. Due to the fact that a Feature ID often needs to be set in the path of a HTTP request, we strongly recommend to use a restricted the set of characters (e.g. those for [Uniform Resource Identifiers (URI)](http://www.ietf.org/rfc/rfc3986.txt)).", "additionalProperties": { "title": "Feature", "type": "object", - "description": "A Feature is used to manage all data and functionality of a Thing that can be clustered in an outlined technical context.", + "description": "A Feature is used to manage all data and functionality of a thing that can be clustered in an outlined technical context.", "additionalProperties": { "type": "object", "description": "The elements of a Feature.", @@ -67,21 +67,21 @@ }, "_revision": { "type": "integer", - "description": "_(read-only)_ The revision is a counter which is incremented on each modification of a Thing." + "description": "_(read-only)_ The revision is a counter which is incremented on each modification of a thing." }, "_created": { "type": "string", - "description": "_(read-only)_ The created timestamp of the Thing in ISO-8601 UTC format. The timestamp is set on creation of a Thing.", + "description": "_(read-only)_ The created timestamp of the thing in ISO-8601 UTC format. The timestamp is set on creation of a thing.", "format": "date-time" }, "_modified": { "type": "string", - "description": "_(read-only)_ The modified timestamp of the Thing in ISO-8601 UTC format. The timestamp is set on each modification of a Thing.", + "description": "_(read-only)_ The modified timestamp of the thing in ISO-8601 UTC format. The timestamp is set on each modification of a thing.", "format": "date-time" }, "_metadata": { "type": "object", - "description": "_(read-only)_ The Metadata of the Thing. This field is not returned by default but must be selected explicitly. The content is a JSON object having the Thing's JSON structure with the difference that the JSON leaves of the Thing are JSON objects containing the metadata." + "description": "_(read-only)_ The Metadata of the thing. This field is not returned by default but must be selected explicitly. The content is a JSON object having the thing's JSON structure with the difference that the JSON leaves of the thing are JSON objects containing the metadata." } }, "required": [ diff --git a/documentation/src/main/resources/openapi/ditto-api-2.yml b/documentation/src/main/resources/openapi/ditto-api-2.yml index 521b4b0456b..18684504337 100644 --- a/documentation/src/main/resources/openapi/ditto-api-2.yml +++ b/documentation/src/main/resources/openapi/ditto-api-2.yml @@ -29,7 +29,7 @@ tags: - name: CloudEvents description: Process CloudEvents in Ditto - name: Connections - description: Manage Connections + description: Manage connections security: - Google: - openid @@ -6818,13 +6818,14 @@ paths: command to process. /connections: get: - summary: Retrieve all Connections + summary: Retrieve all connections description: Returns all connections. security: - DevOpsBasic: [] tags: - Connections parameters: + - $ref: '#/components/parameters/ConnectionFieldsQueryParam' - name: ids-only in: query description: 'When set to true, the request will return the registered ids only and not the whole connections objects.' @@ -6865,7 +6866,7 @@ paths: schema: $ref: '#/components/schemas/AdvancedError' post: - summary: Create a new Connection + summary: Create a new connection description: |- Creates the connection defined in the JSON body. The ID of the connection will be **generated** by the backend. Any `ID` specified in the request body is therefore @@ -6965,14 +6966,15 @@ paths: required: true '/connections/{connectionId}': get: - summary: Retrieve a specific Connection + summary: Retrieve a specific connection description: Returns the connection identified by the `connectionId` path parameter. security: - DevOpsBasic: [] tags: - Connections parameters: - - $ref: '#/components/parameters/connectionIdPathParam' + - $ref: '#/components/parameters/ConnectionIdPathParam' + - $ref: '#/components/parameters/ConnectionFieldsQueryParam' responses: '200': description: The request successfully returned the connection. @@ -7005,14 +7007,14 @@ paths: schema: $ref: '#/components/schemas/AdvancedError' put: - summary: Update a specific Connection registered + summary: Create or update a connection with a specified ID description: Update the connection identified by the `connectionId` path parameter. security: - DevOpsBasic: [] tags: - Connections parameters: - - $ref: '#/components/parameters/connectionIdPathParam' + - $ref: '#/components/parameters/ConnectionIdPathParam' responses: '204': description: The connection was successfully updated. @@ -7079,14 +7081,14 @@ paths: headerMapping: {} required: true delete: - summary: Delete a specific Connection + summary: Delete a specific connection description: Delete the connection identified by the `connectionId` path parameter. security: - DevOpsBasic: [] tags: - Connections parameters: - - $ref: '#/components/parameters/connectionIdPathParam' + - $ref: '#/components/parameters/ConnectionIdPathParam' responses: '204': description: The connection was successfully deleted. @@ -7116,7 +7118,7 @@ paths: $ref: '#/components/schemas/AdvancedError' '/connections/{connectionId}/command': post: - summary: Send a command to a specific Connection. + summary: Send a command to a specific connection description: |- Sends the command specified in the body to the connection identified by the `connectionId` path parameter. @@ -7125,7 +7127,7 @@ paths: tags: - Connections parameters: - - $ref: '#/components/parameters/connectionIdPathParam' + - $ref: '#/components/parameters/ConnectionIdPathParam' responses: '200': description: The command was sent to the connection successfully. @@ -7169,14 +7171,14 @@ paths: required: true '/connections/{connectionId}/status': get: - summary: Retrieve status of a specific Connection + summary: Retrieve status of a specific connection description: Returns the status of the connection identified by the `connectionId` path parameter. security: - DevOpsBasic: [] tags: - Connections parameters: - - $ref: '#/components/parameters/connectionIdPathParam' + - $ref: '#/components/parameters/ConnectionIdPathParam' responses: '200': description: The request successfully returned the connection status. @@ -7210,14 +7212,14 @@ paths: $ref: '#/components/schemas/AdvancedError' '/connections/{connectionId}/metrics': get: - summary: Retrieve metrics of a specific Connection + summary: Retrieve metrics of a specific connection description: Returns the metrics of the connection identified by the `connectionId` path parameter. security: - DevOpsBasic: [] tags: - Connections parameters: - - $ref: '#/components/parameters/connectionIdPathParam' + - $ref: '#/components/parameters/ConnectionIdPathParam' responses: '200': description: The request successfully returned the connection metrics. @@ -7251,7 +7253,7 @@ paths: $ref: '#/components/schemas/AdvancedError' '/connections/{connectionId}/logs': get: - summary: Retrieve logs of a specific Connection + summary: Retrieve logs of a specific connection description: |- Returns the logs of the connection identified by the `connectionId` path parameter. **Before** log entries are generated and returned, logging needs be enabled with the `command` @@ -7262,7 +7264,7 @@ paths: tags: - Connections parameters: - - $ref: '#/components/parameters/connectionIdPathParam' + - $ref: '#/components/parameters/ConnectionIdPathParam' responses: '200': description: The request successfully returned the connection logs. @@ -8158,13 +8160,58 @@ components: required: false schema: type: string - connectionIdPathParam: + ConnectionIdPathParam: name: connectionId in: path description: The ID of the connection required: true schema: type: string + ConnectionFieldsQueryParam: + name: fields + in: query + description: |- + Contains a comma-separated list of fields to be included in the returned + JSON. + + #### Selectable fields + + * `id` + * `name` + * `_revision` + + Specifically selects the revision of the connection. The revision is a counter, which is incremented on each modification of a connection. + + * `_created` + + Specifically selects the created timestamp of the connection in ISO-8601 UTC format. The timestamp is set on creation of a connection. + + * `_modified` + + Specifically selects the modified timestamp of the connection in ISO-8601 UTC format. The timestamp is set on each modification of a connection. + + * `connectionType` + * `connectionStatus` + * `credentials` + * `uri` + * `sources` + * `targets` + * `sshTunnel` + * `clientCount` + * `failoverEnabled` + * `validateCertificates` + * `processorPoolSize` + * `specificConfig` + * `mappingDefinitions` + * `tags` + * `ca` + + #### Examples + + * `?fields=id,_revision,sources` + required: false + schema: + type: string schemas: Error: properties: diff --git a/documentation/src/main/resources/openapi/sources/api-2-index.yml b/documentation/src/main/resources/openapi/sources/api-2-index.yml index c27374cb8ef..00a8df3365e 100644 --- a/documentation/src/main/resources/openapi/sources/api-2-index.yml +++ b/documentation/src/main/resources/openapi/sources/api-2-index.yml @@ -41,7 +41,7 @@ tags: - name: CloudEvents description: Process CloudEvents in Ditto - name: Connections - description: Manage Connections + description: Manage connections security: - Google: @@ -271,8 +271,10 @@ components: $ref: "./parameters/thingIdPathParam.yml" TimeoutParam: $ref: "./parameters/timeoutParam.yml" - connectionIdPathParam: + ConnectionIdPathParam: $ref: './parameters/connectionIdPathParam.yml' + ConnectionFieldsQueryParam: + $ref: "./parameters/connectionFieldsQueryParam.yml" schemas: Error: diff --git a/documentation/src/main/resources/openapi/sources/parameters/connectionFieldsQueryParam.yml b/documentation/src/main/resources/openapi/sources/parameters/connectionFieldsQueryParam.yml new file mode 100644 index 00000000000..72219de913b --- /dev/null +++ b/documentation/src/main/resources/openapi/sources/parameters/connectionFieldsQueryParam.yml @@ -0,0 +1,54 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +name: fields +in: query +description: |- + Contains a comma-separated list of fields to be included in the returned + JSON. + + #### Selectable fields + + * `id` + * `name` + * `_revision` + + Specifically selects the revision of the connection. The revision is a counter, which is incremented on each modification of a connection. + + * `_created` + + Specifically selects the created timestamp of the connection in ISO-8601 UTC format. The timestamp is set on creation of a connection. + + * `_modified` + + Specifically selects the modified timestamp of the connection in ISO-8601 UTC format. The timestamp is set on each modification of a connection. + + * `connectionType` + * `connectionStatus` + * `credentials` + * `uri` + * `sources` + * `targets` + * `sshTunnel` + * `clientCount` + * `failoverEnabled` + * `validateCertificates` + * `processorPoolSize` + * `specificConfig` + * `mappingDefinitions` + * `tags` + * `ca` + + #### Examples + + * `?fields=id,_revision,sources` +required: false +schema: + type: string \ No newline at end of file diff --git a/documentation/src/main/resources/openapi/sources/paths/connections/command.yml b/documentation/src/main/resources/openapi/sources/paths/connections/command.yml index cd0ec278f05..58286fad517 100644 --- a/documentation/src/main/resources/openapi/sources/paths/connections/command.yml +++ b/documentation/src/main/resources/openapi/sources/paths/connections/command.yml @@ -9,7 +9,7 @@ # # SPDX-License-Identifier: EPL-2.0 post: - summary: Send a command to a specific Connection. + summary: Send a command to a specific connection description: |- Sends the command specified in the body to the connection identified by the `connectionId` path parameter. diff --git a/documentation/src/main/resources/openapi/sources/paths/connections/connectionId.yml b/documentation/src/main/resources/openapi/sources/paths/connections/connectionId.yml index 41c16566236..a76a2affb06 100644 --- a/documentation/src/main/resources/openapi/sources/paths/connections/connectionId.yml +++ b/documentation/src/main/resources/openapi/sources/paths/connections/connectionId.yml @@ -9,7 +9,7 @@ # # SPDX-License-Identifier: EPL-2.0 get: - summary: Retrieve a specific Connection + summary: Retrieve a specific connection description: |- Returns the connection identified by the `connectionId` path parameter. security: @@ -18,6 +18,7 @@ get: - Connections parameters: - $ref: '../../parameters/connectionIdPathParam.yml' + - $ref: '../../parameters/connectionFieldsQueryParam.yml' responses: '200': description: The request successfully returned the connection. @@ -52,7 +53,7 @@ get: schema: $ref: '../../schemas/errors/advancedError.yml' put: - summary: Update a specific Connection registered + summary: Create or update a connection with a specified ID description: |- Update the connection identified by the `connectionId` path parameter. security: @@ -142,7 +143,7 @@ put: } required: true delete: - summary: Delete a specific Connection + summary: Delete a specific connection description: |- Delete the connection identified by the `connectionId` path parameter. security: diff --git a/documentation/src/main/resources/openapi/sources/paths/connections/connections.yml b/documentation/src/main/resources/openapi/sources/paths/connections/connections.yml index dfc2b5372a3..310948336a3 100644 --- a/documentation/src/main/resources/openapi/sources/paths/connections/connections.yml +++ b/documentation/src/main/resources/openapi/sources/paths/connections/connections.yml @@ -9,7 +9,7 @@ # # SPDX-License-Identifier: EPL-2.0 get: - summary: Retrieve all Connections + summary: Retrieve all connections description: |- Returns all connections. security: @@ -17,6 +17,7 @@ get: tags: - Connections parameters: + - $ref: '../../parameters/connectionFieldsQueryParam.yml' - name: ids-only in: query description: |- @@ -60,7 +61,7 @@ get: schema: $ref: '../../schemas/errors/advancedError.yml' post: - summary: Create a new Connection + summary: Create a new connection description: |- Creates the connection defined in the JSON body. The ID of the connection will be **generated** by the backend. Any `ID` specified in the request body is therefore diff --git a/documentation/src/main/resources/openapi/sources/paths/connections/logs.yml b/documentation/src/main/resources/openapi/sources/paths/connections/logs.yml index fa2c6d03abe..605b1bb7d1f 100644 --- a/documentation/src/main/resources/openapi/sources/paths/connections/logs.yml +++ b/documentation/src/main/resources/openapi/sources/paths/connections/logs.yml @@ -9,7 +9,7 @@ # # SPDX-License-Identifier: EPL-2.0 get: - summary: Retrieve logs of a specific Connection + summary: Retrieve logs of a specific connection description: |- Returns the logs of the connection identified by the `connectionId` path parameter. **Before** log entries are generated and returned, logging needs be enabled with the `command` diff --git a/documentation/src/main/resources/openapi/sources/paths/connections/metrics.yml b/documentation/src/main/resources/openapi/sources/paths/connections/metrics.yml index a9b0a10ae16..be0f5846cc7 100644 --- a/documentation/src/main/resources/openapi/sources/paths/connections/metrics.yml +++ b/documentation/src/main/resources/openapi/sources/paths/connections/metrics.yml @@ -9,7 +9,7 @@ # # SPDX-License-Identifier: EPL-2.0 get: - summary: Retrieve metrics of a specific Connection + summary: Retrieve metrics of a specific connection description: |- Returns the metrics of the connection identified by the `connectionId` path parameter. security: diff --git a/documentation/src/main/resources/openapi/sources/paths/connections/status.yml b/documentation/src/main/resources/openapi/sources/paths/connections/status.yml index 34f93bfd76b..b7f3296734f 100644 --- a/documentation/src/main/resources/openapi/sources/paths/connections/status.yml +++ b/documentation/src/main/resources/openapi/sources/paths/connections/status.yml @@ -9,7 +9,7 @@ # # SPDX-License-Identifier: EPL-2.0 get: - summary: Retrieve status of a specific Connection + summary: Retrieve status of a specific connection description: |- Returns the status of the connection identified by the `connectionId` path parameter. security: diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/actors/AbstractConnectionsRetrievalActor.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/actors/AbstractConnectionsRetrievalActor.java index 0805ae353ea..6c4dde22a86 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/actors/AbstractConnectionsRetrievalActor.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/actors/AbstractConnectionsRetrievalActor.java @@ -24,6 +24,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import javax.annotation.Nullable; + import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.base.model.headers.WithDittoHeaders; @@ -46,6 +48,7 @@ import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig; import org.eclipse.ditto.json.JsonArray; import org.eclipse.ditto.json.JsonCollectors; +import org.eclipse.ditto.json.JsonFieldSelector; import akka.actor.AbstractActor; import akka.actor.ActorRef; @@ -111,7 +114,10 @@ protected void retrieveConnectionsById(final RetrieveAllConnectionIdsResponse al retrieveConnections(connectionIds .stream() .map(ConnectionId::of) - .toList(), initialCommand.getDittoHeaders()) + .toList(), + initialCommand.getSelectedFields().orElse(null), + initialCommand.getDittoHeaders() + ) .thenAccept(retrieveConnectionsResponse -> { sender.tell(retrieveConnectionsResponse, getSelf()); stop(); @@ -147,13 +153,15 @@ private void handleRetrieveConnections(final RetrieveConnections retrieveConnect } private CompletionStage retrieveConnections( - final Collection connectionIds, final DittoHeaders dittoHeaders) { + final Collection connectionIds, + @Nullable final JsonFieldSelector selectedFields, + final DittoHeaders dittoHeaders) { checkNotNull(connectionIds, "connectionIds"); checkNotNull(dittoHeaders, "dittoHeaders"); final List> completableFutures = connectionIds.parallelStream() - .map(connectionId -> retrieveConnection(RetrieveConnection.of(connectionId, dittoHeaders))) + .map(connectionId -> retrieveConnection(RetrieveConnection.of(connectionId, selectedFields, dittoHeaders))) .map(CompletionStage::toCompletableFuture) .toList(); diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsParameter.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsParameter.java index 93da53e6d70..d07da3c7272 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsParameter.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsParameter.java @@ -22,7 +22,13 @@ public enum ConnectionsParameter { * Request parameter for doing a dry-run before creating a connection. */ DRY_RUN("dry-run"), - IDS_ONLY("ids-only"); + + IDS_ONLY("ids-only"), + + /** + * Request parameter for including only the selected fields in the Connection JSON document(s). + */ + FIELDS("fields"); private final String parameterValue; diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsRoute.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsRoute.java index 631d66c73a4..b2f26de0ced 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsRoute.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/endpoints/routes/connections/ConnectionsRoute.java @@ -51,8 +51,10 @@ import org.eclipse.ditto.gateway.service.endpoints.routes.AbstractRoute; import org.eclipse.ditto.gateway.service.endpoints.routes.RouteBaseProperties; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonFieldSelector; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; import akka.http.javadsl.model.MediaTypes; import akka.http.javadsl.server.PathMatchers; @@ -142,11 +144,25 @@ private Route connections(final RequestContext ctx, final DittoHeaders dittoHead return pathEndOrSingleSlash(() -> concat( get(() -> // GET /connections?ids-only=false - parameterOptional(ConnectionsParameter.IDS_ONLY.toString(), idsOnly -> handlePerRequest(ctx, - RetrieveConnections.newInstance(idsOnly.map(Boolean::valueOf).orElse(false), - dittoHeaders) - )) - + parameterOptional(ConnectionsParameter.IDS_ONLY.toString(), idsOnly -> + parameterOptional(ConnectionsParameter.FIELDS.toString(), fieldsString -> + { + final Optional selectedFields = + calculateSelectedFields(fieldsString); + return handlePerRequest(ctx, RetrieveConnections.newInstance( + idsOnly.map(Boolean::valueOf).orElseGet(() -> selectedFields + .filter(sf -> sf.getPointers().size() == 1 && + sf.getPointers().contains( + JsonPointer.of("id") + ) + ).isPresent() + ), + selectedFields.orElse(null), + dittoHeaders) + ); + } + ) + ) ), post(() -> // POST /connections?dry-run= parameterOptional(ConnectionsParameter.DRY_RUN.toString(), dryRun -> @@ -198,13 +214,17 @@ private Route connectionsEntry(final RequestContext ctx, payloadSource -> handlePerRequest(ctx, dittoHeaders, payloadSource, payloadJsonString -> ModifyConnection.of( - buildConnectionForPut(connectionId, - payloadJsonString), + buildConnectionForPut(connectionId, payloadJsonString), dittoHeaders)) ) ), get(() -> // GET /connections/ - handlePerRequest(ctx, RetrieveConnection.of(connectionId, dittoHeaders)) + parameterOptional(ConnectionsParameter.FIELDS.toString(), fieldsString -> + handlePerRequest(ctx, RetrieveConnection.of(connectionId, + calculateSelectedFields(fieldsString).orElse(null), + dittoHeaders) + ) + ) ), delete(() -> // DELETE /connections/ handlePerRequest(ctx, DeleteConnection.of(connectionId, dittoHeaders)) diff --git a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java index 0c956f8b7b5..6c5bf984dba 100755 --- a/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java +++ b/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java @@ -89,7 +89,7 @@ public static RetrievePolicy of(final PolicyId policyId, final DittoHeaders ditt * * @param policyId the ID of a single Policy to be retrieved by this command. * @param dittoHeaders the optional command headers of the request. - * @param selectedFields the fields of the JSON representation of the Thing to retrieve. + * @param selectedFields the fields of the JSON representation of the Policy to retrieve. * @return a Command for retrieving the Policy with the {@code policyId} as its ID which is readable from the passed * authorization context. * @throws NullPointerException if any argument is {@code null}. diff --git a/things/service/src/main/resources/things.conf b/things/service/src/main/resources/things.conf index e0741e2ae6c..0e3e3237e74 100755 --- a/things/service/src/main/resources/things.conf +++ b/things/service/src/main/resources/things.conf @@ -9,7 +9,7 @@ ditto { "org.eclipse.ditto.policies.enforcement.pre.CreationRestrictionPreEnforcer" ] signal-transformers-provider.extension-config.signal-transformers = [ - "org.eclipse.ditto.things.service.enforcement.pre.ModifyToCreateThingTransformer", // always keep this as first transformer in order to guarantee that all following transformers know that the command is creating a policy instead of modifying it + "org.eclipse.ditto.things.service.enforcement.pre.ModifyToCreateThingTransformer", // always keep this as first transformer in order to guarantee that all following transformers know that the command is creating a thing instead of modifying it "org.eclipse.ditto.things.service.signaltransformation.placeholdersubstitution.ThingsPlaceholderSubstitution" ] snapshot-adapter = "org.eclipse.ditto.things.service.persistence.serializer.ThingMongoSnapshotAdapter" From 76fabb35097e5cce7d28664169ffafa069aea3fd Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Mon, 30 Jan 2023 09:18:54 +0100 Subject: [PATCH 027/173] fixed that a missing (deleted) referenced policy of a policy import caused logging ERRORs in the BackgroundSyncStream Signed-off-by: Thomas Jaeckle --- .../write/streaming/BackgroundSyncStream.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/write/streaming/BackgroundSyncStream.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/write/streaming/BackgroundSyncStream.java index 24408f24ad3..4842c267655 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/write/streaming/BackgroundSyncStream.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/write/streaming/BackgroundSyncStream.java @@ -36,6 +36,7 @@ import org.eclipse.ditto.policies.model.PolicyConstants; import org.eclipse.ditto.policies.model.PolicyId; import org.eclipse.ditto.policies.model.PolicyImport; +import org.eclipse.ditto.policies.model.signals.commands.exceptions.PolicyNotAccessibleException; import org.eclipse.ditto.things.model.ThingConstants; import org.eclipse.ditto.things.model.ThingId; import org.eclipse.ditto.thingsearch.service.persistence.write.model.Metadata; @@ -53,7 +54,7 @@ */ public final class BackgroundSyncStream { - private static Logger LOGGER = DittoLoggerFactory.getThreadSafeLogger(BackgroundSyncStream.class); + private static final Logger LOGGER = DittoLoggerFactory.getThreadSafeLogger(BackgroundSyncStream.class); private static final ThingId EMPTY_THING_ID = ThingId.of(LowerBound.emptyEntityId(ThingConstants.ENTITY_TYPE)); private static final PolicyId EMPTY_POLICY_ID = PolicyId.of(LowerBound.emptyEntityId(PolicyConstants.ENTITY_TYPE)); @@ -186,7 +187,7 @@ private Source emitUnlessConsistent(final Metadata persisted, .orElseGet(() -> CompletableFuture.completedStage(true)); return Source.completionStage(consistencyCheckCs) .flatMapConcat(policiesAreConsistent -> { - if (policiesAreConsistent) { + if (Boolean.TRUE.equals(policiesAreConsistent)) { return Source.empty(); } else { return Source.single(indexed.invalidateCaches(false, true)).log("PoliciesInconsistent"); @@ -283,13 +284,17 @@ private CompletionStage isPolicyTagUpToDate(final PolicyTag policyTag) final String message = String.format( "Error in background sync stream when trying to retrieve policy revision of " + "Policy with ID <%s>", policyId); - LOGGER.error(message, error); + LOGGER.warn(message, error); return false; } else if (response instanceof SudoRetrievePolicyRevisionResponse retrieveResponse) { final long revision = retrieveResponse.getRevision(); return policyTag.getRevision() == revision; + } else if (response instanceof PolicyNotAccessibleException) { + LOGGER.info("Policy with ID <{}> is/was no longer accessible when trying to retrieve policy " + + "revision of Policy", policyId); + return false; } else { - LOGGER.error("Unexpected message in background sync stream when trying to retrieve policy " + + LOGGER.warn("Unexpected message in background sync stream when trying to retrieve policy " + "revision of Policy with ID <{}>. Expected <{}> but got <{}>.", policyId, SudoRetrievePolicyRevisionResponse.class, response.getClass()); return false; @@ -305,12 +310,16 @@ private CompletionStage> retrievePolicy(final PolicyId policyId if (error != null) { final String message = String.format("Error in background sync stream when trying to " + "retrieve policy with ID <%s>", policyId); - LOGGER.error(message, error); + LOGGER.warn(message, error); return Optional.empty(); } else if (response instanceof SudoRetrievePolicyResponse retrieveResponse) { return Optional.of(retrieveResponse.getPolicy()); + } else if (response instanceof PolicyNotAccessibleException) { + LOGGER.info("Policy with ID <{}> is/was no longer accessible when trying to retrieve policy", + policyId); + return Optional.empty(); } else { - LOGGER.error("Unexpected message in background sync stream when trying to retrieve policy " + + LOGGER.warn("Unexpected message in background sync stream when trying to retrieve policy " + "with ID <{}>. Expected <{}> but got <{}>.", policyId, SudoRetrievePolicyRevisionResponse.class, response.getClass()); return Optional.empty(); From 25f6a1e7f11b816a30ce650da650cfd0e5a21336 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Tue, 31 Jan 2023 15:43:53 +0100 Subject: [PATCH 028/173] stabilized DittoProtocolSubImpl when unresolved placeholders were contained in acknowledgement labels Signed-off-by: Thomas Jaeckle --- .../grafana-datasources/datasource.yaml | 39 +++++++++++++++++++ .../pubsubthings/DittoProtocolSubImpl.java | 18 ++++++--- 2 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 deployment/operations/grafana-datasources/datasource.yaml diff --git a/deployment/operations/grafana-datasources/datasource.yaml b/deployment/operations/grafana-datasources/datasource.yaml new file mode 100644 index 00000000000..14f5b356dd7 --- /dev/null +++ b/deployment/operations/grafana-datasources/datasource.yaml @@ -0,0 +1,39 @@ +# config file version +apiVersion: 1 + +# list of datasources to insert/update depending +# whats available in the database +datasources: + # + # Prometheus datasource + # +- name: prometheus + # datasource type. Required + type: prometheus + # access mode. direct or proxy. Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://prometheus:9090 + # allow users to edit datasources from the UI. + editable: false + +- name: elasticsearch + # datasource type. Required + type: elasticsearch + # access mode. direct or proxy. Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://elasticsearch:9200 + # allow users to edit datasources from the UI. + editable: false + database: "[logstash-]YYYY.MM.DD" + # fields that will be converted to json and stored in json_data + jsonData: + interval: Daily + timeField: "@timestamp" + esVersion: '8.0.0' + logMessageField: message diff --git a/internal/utils/pubsub-things/src/main/java/org/eclipse/ditto/internal/utils/pubsubthings/DittoProtocolSubImpl.java b/internal/utils/pubsub-things/src/main/java/org/eclipse/ditto/internal/utils/pubsubthings/DittoProtocolSubImpl.java index 80fde2bfee7..661ec64a1db 100644 --- a/internal/utils/pubsub-things/src/main/java/org/eclipse/ditto/internal/utils/pubsubthings/DittoProtocolSubImpl.java +++ b/internal/utils/pubsub-things/src/main/java/org/eclipse/ditto/internal/utils/pubsubthings/DittoProtocolSubImpl.java @@ -25,6 +25,9 @@ import javax.annotation.Nullable; import org.eclipse.ditto.base.model.acks.AcknowledgementLabel; +import org.eclipse.ditto.base.model.acks.AcknowledgementLabelInvalidException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.internal.utils.pubsub.DistributedAcks; import org.eclipse.ditto.internal.utils.pubsub.DistributedSub; import org.eclipse.ditto.internal.utils.pubsub.StreamingType; @@ -131,9 +134,11 @@ public CompletionStage declareAcknowledgementLabels( return CompletableFuture.completedFuture(null); } - // don't complete the future with the exception this method emits as this is a bug in Ditto which we must escalate - // via the actor supervision strategy - ensureAcknowledgementLabelsAreFullyResolved(acknowledgementLabels); + try { + ensureAcknowledgementLabelsAreFullyResolved(acknowledgementLabels); + } catch (final DittoRuntimeException dre) { + return CompletableFuture.failedStage(dre); + } return distributedAcks.declareAcknowledgementLabels(acknowledgementLabels, subscriber, group) .thenApply(ack -> null); @@ -144,9 +149,10 @@ private static void ensureAcknowledgementLabelsAreFullyResolved(final Collection .filter(Predicate.not(AcknowledgementLabel::isFullyResolved)) .findFirst() .ifPresent(ackLabel -> { - // if this happens, this is a bug in the Ditto codebase! at this point the AckLabel must be resolved - throw new IllegalArgumentException("AcknowledgementLabel was not fully resolved while " + - "trying to declare it: " + ackLabel); + throw AcknowledgementLabelInvalidException.of(ackLabel, + "AcknowledgementLabel was not fully resolved while trying to declare it", + null, + DittoHeaders.empty()); }); } From 44ab2a3dd994f6230cf98aaf89226f9fbea09b68 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Tue, 31 Jan 2023 16:30:39 +0100 Subject: [PATCH 029/173] updated github repository locations to new "eclipse-ditto" org Signed-off-by: Thomas Jaeckle --- NOTICE.md | 2 +- README.md | 4 +- RELEASE.md | 8 +- .../service/mapping/CloudEventsMapper.java | 2 +- .../src/main/resources/_data/topnav.yml | 6 +- ...example-demonstrating-rest-to-websocket.md | 4 +- .../_posts/2018-04-25-connectivity-service.md | 4 +- .../2018-10-16-example-mqtt-bidirectional.md | 4 +- .../2019-12-12-release-announcement-100.md | 2 +- .../2020-04-16-connecting-to-ttn-via-mqtt.md | 2 +- ...2020-10-08-asynchronous-client-creation.md | 2 +- .../2021-03-22-azure-iot-hub-integration.md | 2 +- .../_posts/2022-03-03-wot-integration.md | 2 +- .../DADR-0004-signal-enrichment.md | 2 +- .../DADR-0007-concierge-removal.md | 6 +- .../architecture-services-connectivity.md | 6 +- .../ditto/architecture-services-policies.md | 6 +- .../architecture-services-things-search.md | 4 +- .../ditto/architecture-services-things.md | 6 +- .../pages/ditto/basic-namespaces-and-names.md | 6 +- .../ditto/basic-wot-integration-example.md | 2 +- .../pages/ditto/basic-wot-integration.md | 4 +- .../resources/pages/ditto/client-sdk-java.md | 2 +- .../pages/ditto/client-sdk-javascript.md | 8 +- .../pages/ditto/connectivity-hmac-signing.md | 4 +- .../ditto/connectivity-manage-connections.md | 2 +- .../pages/ditto/connectivity-mapping.md | 6 +- .../connectivity-protocol-bindings-kafka2.md | 2 +- .../main/resources/pages/ditto/feedback.md | 4 +- .../pages/ditto/installation-extending.md | 2 +- .../pages/ditto/installation-operating.md | 26 ++--- .../pages/ditto/installation-running.md | 2 +- .../pages/ditto/intro-hello-world.md | 2 +- .../resources/pages/ditto/presentations.md | 2 +- .../pages/ditto/release_notes_010M3.md | 8 +- .../pages/ditto/release_notes_020M1.md | 8 +- .../pages/ditto/release_notes_030M1.md | 10 +- .../pages/ditto/release_notes_030M2.md | 12 +-- .../pages/ditto/release_notes_080.md | 4 +- .../pages/ditto/release_notes_080M1.md | 16 +-- .../pages/ditto/release_notes_080M2.md | 18 ++-- .../pages/ditto/release_notes_080M3.md | 10 +- .../pages/ditto/release_notes_090.md | 14 +-- .../pages/ditto/release_notes_090M1.md | 14 +-- .../pages/ditto/release_notes_090M2.md | 18 ++-- .../pages/ditto/release_notes_100.md | 8 +- .../pages/ditto/release_notes_100M1a.md | 24 ++--- .../pages/ditto/release_notes_100M2.md | 18 ++-- .../pages/ditto/release_notes_110.md | 36 +++---- .../pages/ditto/release_notes_111.md | 6 +- .../pages/ditto/release_notes_112.md | 12 +-- .../pages/ditto/release_notes_113.md | 16 +-- .../pages/ditto/release_notes_115.md | 4 +- .../pages/ditto/release_notes_120.md | 22 ++--- .../pages/ditto/release_notes_121.md | 4 +- .../pages/ditto/release_notes_130.md | 24 ++--- .../pages/ditto/release_notes_140.md | 26 ++--- .../pages/ditto/release_notes_150.md | 34 +++---- .../pages/ditto/release_notes_151.md | 2 +- .../pages/ditto/release_notes_200.md | 52 +++++----- .../pages/ditto/release_notes_201.md | 12 +-- .../pages/ditto/release_notes_210.md | 60 ++++++------ .../pages/ditto/release_notes_211.md | 16 +-- .../pages/ditto/release_notes_212.md | 10 +- .../pages/ditto/release_notes_213.md | 4 +- .../pages/ditto/release_notes_220.md | 24 ++--- .../pages/ditto/release_notes_221.md | 8 +- .../pages/ditto/release_notes_230.md | 20 ++-- .../pages/ditto/release_notes_231.md | 10 +- .../pages/ditto/release_notes_232.md | 10 +- .../pages/ditto/release_notes_240.md | 32 +++--- .../pages/ditto/release_notes_241.md | 6 +- .../pages/ditto/release_notes_242.md | 4 +- .../pages/ditto/release_notes_300.md | 98 +++++++++---------- .../2018_02_07-virtualiot-meetup/index.html | 2 +- .../2018_05_23-meetup-iot-hessen/index.html | 4 +- .../index.html | 6 +- .../index.html | 2 +- .../index.html | 4 +- .../index.html | 2 +- .../index.html | 6 +- .../index.html | 4 +- .../2021_06_ditto-20-overview/index.html | 4 +- .../slides/2021_06_ditto-in-20-min/index.html | 4 +- .../index.html | 8 +- .../index.html | 4 +- .../slides/2022_10_ditto-and-wot/index.html | 4 +- .../slides/2023_01_ditto-in-30-min/index.html | 4 +- legal/NOTICE.md | 2 +- pom.xml | 2 +- 90 files changed, 472 insertions(+), 472 deletions(-) diff --git a/NOTICE.md b/NOTICE.md index 3e022ace3d9..0f7d66929c6 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -23,7 +23,7 @@ SPDX-License-Identifier: EPL-2.0 # Source Code -* https://github.com/eclipse/ditto +* https://github.com/eclipse-ditto/ditto # Third-party Content diff --git a/README.md b/README.md index 71564ff8b25..ee6d28b749c 100755 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ # Eclipse Ditto™ [![Join the chat at https://gitter.im/eclipse/ditto](https://badges.gitter.im/eclipse/ditto.svg)](https://gitter.im/eclipse/ditto?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Build Status](https://github.com/eclipse/ditto/workflows/build/badge.svg)](https://github.com/eclipse/ditto/actions?query=workflow%3Abuild) +[![Build Status](https://github.com/eclipse-ditto/ditto/workflows/build/badge.svg)](https://github.com/eclipse-ditto/ditto/actions?query=workflow%3Abuild) [![Maven Central](https://img.shields.io/maven-central/v/org.eclipse.ditto/ditto?label=maven)](https://search.maven.org/search?q=g:org.eclipse.ditto) [![Docker pulls](https://img.shields.io/docker/pulls/eclipse/ditto-things.svg)](https://hub.docker.com/search?q=eclipse%2Fditto&type=image) [![License](https://img.shields.io/badge/License-EPL%202.0-green.svg)](https://opensource.org/licenses/EPL-2.0) @@ -41,7 +41,7 @@ In order to start up Ditto via *Docker Compose*, you'll need: * at least 2 CPU cores which can be used by Docker * at least 4 GB of RAM which can be used by Docker -You also have other possibilities to run Ditto, please have a look [here](https://github.com/eclipse/ditto/tree/master/deployment) to explore them. +You also have other possibilities to run Ditto, please have a look [here](https://github.com/eclipse-ditto/ditto/tree/master/deployment) to explore them. ### Start Ditto diff --git a/RELEASE.md b/RELEASE.md index b986c398bd4..406b6573eba 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -18,17 +18,17 @@ Ditto releases are tracked and planned here: https://projects.eclipse.org/projec * New features, changes, bug fixes to last release / milestone release * Add migration notes (if there are any) * Write a Blog post announcement, e.g. like for: https://www.eclipse.org/ditto/2022-12-16-release-announcement-310.html -* Close GitHub milestone (and assign all Issues/PRs which were still included in that milestone): https://github.com/eclipse/ditto/milestones -* Create a GitHub release: https://github.com/eclipse/ditto/releases (based on the Tags which was pushed during release job) +* Close GitHub milestone (and assign all Issues/PRs which were still included in that milestone): https://github.com/eclipse-ditto/ditto/milestones +* Create a GitHub release: https://github.com/eclipse-ditto/ditto/releases (based on the Tags which was pushed during release job) * Write a mail to the "ditto-dev" mailing list * Tweet about it ;) * Set binary compatibility check version to the new public release. Delete all exclusions and module-level deactivation of japi-cmp plugin except for *.internal packages. -* Update https://github.com/eclipse/ditto/blob/master/SECURITY.md with the supported versions to receive security fixes +* Update https://github.com/eclipse-ditto/ditto/blob/master/SECURITY.md with the supported versions to receive security fixes * For major+minor versions: * Create a "release" branch"release-" from the released git tag * needed to build the documentation from * required for bugfixes to build a bugfix release for the affected minor version - * Add the new version to the documentation config: https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/_config.yml#L114 + * Add the new version to the documentation config: https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/_config.yml#L114 * Adjust the "website" CI jobs to also build the newly added branch: * https://ci.eclipse.org/ditto/view/Website/ * https://ci.eclipse.org/ditto/view/Website/job/website-build-and-deploy-fast/ will build the latest released minor version + "master" (development) version diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/CloudEventsMapper.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/CloudEventsMapper.java index b863c083fe7..1c5d913dddb 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/CloudEventsMapper.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/CloudEventsMapper.java @@ -58,7 +58,7 @@ public final class CloudEventsMapper extends AbstractMessageMapper { private static final String TYPE = "type"; private static final String OUTBOUNDTYPE = "org.eclipse.ditto.outbound"; private static final String OUTBOUNDSPECVERSION = "1.0"; - private static final String OUTBOUNDSOURCE = "https://github.com/eclipse/ditto"; + private static final String OUTBOUNDSOURCE = "https://github.com/eclipse-ditto/ditto"; private static final String DATA = "data"; private static final String DATA_BASE64 = "data_base64"; private static final String OUTBOUND_DATA_CONTENT_TYPE = "datacontenttype"; diff --git a/documentation/src/main/resources/_data/topnav.yml b/documentation/src/main/resources/_data/topnav.yml index 4bf938138ec..7a473874854 100644 --- a/documentation/src/main/resources/_data/topnav.yml +++ b/documentation/src/main/resources/_data/topnav.yml @@ -13,15 +13,15 @@ topnav: - title: image: GitHub-Mark-Light-32px.png alt: Sources at GitHub - external_url: https://github.com/eclipse/ditto + external_url: https://github.com/eclipse-ditto/ditto - title: SDKs image: GitHub-Mark-Light-32px.png alt: SDK sources at GitHub - external_url: https://github.com/eclipse/ditto-clients + external_url: https://github.com/eclipse-ditto/ditto-clients - title: examples image: GitHub-Mark-Light-32px.png alt: Example sources at GitHub - external_url: https://github.com/eclipse/ditto-examples + external_url: https://github.com/eclipse-ditto/ditto-examples #Topnav dropdowns topnav_dropdowns: diff --git a/documentation/src/main/resources/_posts/2018-01-15-example-demonstrating-rest-to-websocket.md b/documentation/src/main/resources/_posts/2018-01-15-example-demonstrating-rest-to-websocket.md index e8a4ba79ba1..62e96d14bc3 100644 --- a/documentation/src/main/resources/_posts/2018-01-15-example-demonstrating-rest-to-websocket.md +++ b/documentation/src/main/resources/_posts/2018-01-15-example-demonstrating-rest-to-websocket.md @@ -11,7 +11,7 @@ toc: false --- There's a new example showing how to combine the REST and WebSocket API -over at the [Eclipse Ditto examples repository](https://github.com/eclipse/ditto-examples/tree/master/rest-to-websocket). +over at the [Eclipse Ditto examples repository](https://github.com/eclipse-ditto/ditto-examples/tree/master/rest-to-websocket). Right from the project's description: >This example shows how to leverage the powers of combining the REST and @@ -31,7 +31,7 @@ Right from the project's description: href="https://raw.githubusercontent.com/eclipse/ditto-examples/master/rest-to-websocket/docs/images/make-coffee.gif" alt="Eclipse Ditto REST to WebSocket example gif" max-width=800 -caption="Source: https://github.com/eclipse/ditto-examples" %} +caption="Source: https://github.com/eclipse-ditto/ditto-examples" %} If you have any wishes, improvements, are missing something or just want to get in touch with us, you can use one of diff --git a/documentation/src/main/resources/_posts/2018-04-25-connectivity-service.md b/documentation/src/main/resources/_posts/2018-04-25-connectivity-service.md index b466b335581..d8f87c6ab18 100644 --- a/documentation/src/main/resources/_posts/2018-04-25-connectivity-service.md +++ b/documentation/src/main/resources/_posts/2018-04-25-connectivity-service.md @@ -24,8 +24,8 @@ That worked quite well, but still had some issues: Our current implementation focus lies on two GitHub issues resolving those problems: -* [Enhance existing AMQP-bridge with AMQP 0.9.1 connectivity](https://github.com/eclipse/ditto/issues/129) -* [Support mapping arbitrary message payloads in AMQP-bridge](https://github.com/eclipse/ditto/issues/130) +* [Enhance existing AMQP-bridge with AMQP 0.9.1 connectivity](https://github.com/eclipse-ditto/ditto/issues/129) +* [Support mapping arbitrary message payloads in AMQP-bridge](https://github.com/eclipse-ditto/ditto/issues/130) ## Changes and Enhancements diff --git a/documentation/src/main/resources/_posts/2018-10-16-example-mqtt-bidirectional.md b/documentation/src/main/resources/_posts/2018-10-16-example-mqtt-bidirectional.md index fc03b528fa6..569cb507301 100644 --- a/documentation/src/main/resources/_posts/2018-10-16-example-mqtt-bidirectional.md +++ b/documentation/src/main/resources/_posts/2018-10-16-example-mqtt-bidirectional.md @@ -20,7 +20,7 @@ which was released recently with milestone [0.8.0-M2](2018-09-27-milestone-annou On his journey into digital twin land he made a great example with an ESP8266 powered board connected via an MQTT broker to Ditto and published it to the -[Eclipse Ditto examples repository](https://github.com/eclipse/ditto-examples/tree/master/mqtt-bidirectional): +[Eclipse Ditto examples repository](https://github.com/eclipse-ditto/ditto-examples/tree/master/mqtt-bidirectional): > This example is about how to communicate between device and solution in a two way pattern through Ditto using MQTT. This means we will add a policy, a thing and a MQTT connection to Ditto. @@ -40,7 +40,7 @@ to Ditto and published it to the href="https://raw.githubusercontent.com/eclipse/ditto-examples/master/mqtt-bidirectional/img/diagram.jpg" alt="Eclipse Ditto bidirectional MQTT diagram" max-width=800 -caption="Source: https://github.com/eclipse/ditto-examples" %} +caption="Source: https://github.com/eclipse-ditto/ditto-examples" %} > We will use an Octopus-board with an ESP8266 on it. It has several sensors built in, but for simplicity we will just use it's temperature and altitude sensor. diff --git a/documentation/src/main/resources/_posts/2019-12-12-release-announcement-100.md b/documentation/src/main/resources/_posts/2019-12-12-release-announcement-100.md index f600cd71527..5b892c42708 100644 --- a/documentation/src/main/resources/_posts/2019-12-12-release-announcement-100.md +++ b/documentation/src/main/resources/_posts/2019-12-12-release-announcement-100.md @@ -60,7 +60,7 @@ millions of managed things. The main changes compared to the last release, [0.9.0](release_notes_090.html), are: -* addition of a Java and a JavaScript client SDK in separate [GitHub repo](https://github.com/eclipse/ditto-clients) +* addition of a Java and a JavaScript client SDK in separate [GitHub repo](https://github.com/eclipse-ditto/ditto-clients) * configurable OpenID Connect authorization servers * support for OpenID Connect / OAuth2.0 based authentication in Ditto Java Client * invoking custom foreign HTTP endpoints as a result of events/messages diff --git a/documentation/src/main/resources/_posts/2020-04-16-connecting-to-ttn-via-mqtt.md b/documentation/src/main/resources/_posts/2020-04-16-connecting-to-ttn-via-mqtt.md index 8eb9c448211..9c4d42d14fb 100644 --- a/documentation/src/main/resources/_posts/2020-04-16-connecting-to-ttn-via-mqtt.md +++ b/documentation/src/main/resources/_posts/2020-04-16-connecting-to-ttn-via-mqtt.md @@ -69,7 +69,7 @@ sudo chmod +x /usr/local/bin/docker-compose That is required to get the `docker-compose.yaml` file and other resources required to run Ditto with Docker Compose. ```shell -git clone --depth 1 https://github.com/eclipse/ditto.git +git clone --depth 1 https://github.com/eclipse-ditto/ditto.git ``` diff --git a/documentation/src/main/resources/_posts/2020-10-08-asynchronous-client-creation.md b/documentation/src/main/resources/_posts/2020-10-08-asynchronous-client-creation.md index 8128cf19ac3..a00eff72714 100644 --- a/documentation/src/main/resources/_posts/2020-10-08-asynchronous-client-creation.md +++ b/documentation/src/main/resources/_posts/2020-10-08-asynchronous-client-creation.md @@ -10,7 +10,7 @@ sidebar: false toc: false --- -Before [Ditto Java Client](https://github.com/eclipse/ditto-clients/tree/master/java) 1.3.0, +Before [Ditto Java Client](https://github.com/eclipse-ditto/ditto-clients/tree/master/java) 1.3.0, a client object connects to a configured Ditto back-end during its creation. ```java diff --git a/documentation/src/main/resources/_posts/2021-03-22-azure-iot-hub-integration.md b/documentation/src/main/resources/_posts/2021-03-22-azure-iot-hub-integration.md index 025047de9fd..7e88a7627a1 100644 --- a/documentation/src/main/resources/_posts/2021-03-22-azure-iot-hub-integration.md +++ b/documentation/src/main/resources/_posts/2021-03-22-azure-iot-hub-integration.md @@ -205,5 +205,5 @@ generate a SAS token out of that string. ## Getting started To get started using Azure IoT Hub as a message broker for Eclipse Ditto, the -[Azure IoT Hub Device Simulator Example](https://github.com/eclipse/ditto-examples/tree/master/azure/azure-iot-hub-device-simulator) +[Azure IoT Hub Device Simulator Example](https://github.com/eclipse-ditto/ditto-examples/tree/master/azure/azure-iot-hub-device-simulator) is a good entry point. diff --git a/documentation/src/main/resources/_posts/2022-03-03-wot-integration.md b/documentation/src/main/resources/_posts/2022-03-03-wot-integration.md index 43521bcc84d..41c4d057923 100644 --- a/documentation/src/main/resources/_posts/2022-03-03-wot-integration.md +++ b/documentation/src/main/resources/_posts/2022-03-03-wot-integration.md @@ -191,7 +191,7 @@ e.g. have a builder based API for building new objects or to read a TD/TM from a ``` -Please have a look at the added [ditto-wot-model module](https://github.com/eclipse/ditto/tree/master/wot/model) to find +Please have a look at the added [ditto-wot-model module](https://github.com/eclipse-ditto/ditto/tree/master/wot/model) to find out more about example usage. diff --git a/documentation/src/main/resources/architecture/DADR-0004-signal-enrichment.md b/documentation/src/main/resources/architecture/DADR-0004-signal-enrichment.md index 4a65664e1a0..7717a2c2567 100644 --- a/documentation/src/main/resources/architecture/DADR-0004-signal-enrichment.md +++ b/documentation/src/main/resources/architecture/DADR-0004-signal-enrichment.md @@ -8,7 +8,7 @@ accepted ## Context -Supporting a new feature, the so called [signal enrichment](https://github.com/eclipse/ditto/issues/561), raises a few +Supporting a new feature, the so called [signal enrichment](https://github.com/eclipse-ditto/ditto/issues/561), raises a few questions towards throughput and scalability impact of that new feature. In the current architecture, Ditto internally publishes events (as part of the applied "event sourcing" pattern) for diff --git a/documentation/src/main/resources/architecture/DADR-0007-concierge-removal.md b/documentation/src/main/resources/architecture/DADR-0007-concierge-removal.md index c79c4cfc1a8..2e09671da42 100644 --- a/documentation/src/main/resources/architecture/DADR-0007-concierge-removal.md +++ b/documentation/src/main/resources/architecture/DADR-0007-concierge-removal.md @@ -2,7 +2,7 @@ Date: 12.04.2022 -Related GitHub issue: [#1339](https://github.com/eclipse/ditto/issues/1339) +Related GitHub issue: [#1339](https://github.com/eclipse-ditto/ditto/issues/1339) ## Status @@ -47,10 +47,10 @@ Additional benefits depending on where the authorization / policy enforcement is | Aspect | Approach: authorization at edges (gateway/connectivity) | Approach: authorization in entity services (policies/things) | |----------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Policy enforcer caching | -- **distributed caching** potentially each instance of edge will eventually cache a policy enforced for a specific policy
--- **unpredictable cache sizes** due to missing sharding | +++ **very localized caching**, e.g. a policy only used by one thing is only cached within the `ThingPersistenceActor` which uses the policy | -| Partial event handling [#96](https://github.com/eclipse/ditto/issues/96) | + **edges directly can use policy enforcers** to create multiple "partial events" based on one "source event" | -- **information** regarding how the edges should split up one "source event" into several "partial events" must be **piggybacked as event header** | +| Partial event handling [#96](https://github.com/eclipse-ditto/ditto/issues/96) | + **edges directly can use policy enforcers** to create multiple "partial events" based on one "source event" | -- **information** regarding how the edges should split up one "source event" into several "partial events" must be **piggybacked as event header** | | Live command / message processing | + **edges can directly route authorized live commands / messages** to other interested edges (e.g. gateway WS to connectivity) | - **for live commands / messages one additional "hop"** (only compared to the left column - this "hop" is currently with concierge service also necessary)
via the "things" service has to be done as the "things" service would also authorize live commands/messages. | | Smart channel selection | -- **rather complex state machine logic from concierge must be moved to edges** for supporting the "live channel condition" use cases | ++ **live channel condition** logic can be done completely in `ThingPersistenceActor`, simplifying the implementation a lot | -| Conditional messages [#1363](https://github.com/eclipse/ditto/issues/1363) | - **rather complex implementation required similar to smart channel selection** | + **simple implementation**, can be done completely in `ThingPersistenceActor` | +| Conditional messages [#1363](https://github.com/eclipse-ditto/ditto/issues/1363) | - **rather complex implementation required similar to smart channel selection** | + **simple implementation**, can be done completely in `ThingPersistenceActor` | | ###### | ###### | ###### | | Overall weight | ------ | +++ | diff --git a/documentation/src/main/resources/pages/ditto/architecture-services-connectivity.md b/documentation/src/main/resources/pages/ditto/architecture-services-connectivity.md index 958ad7f5fb8..102f8fe3660 100644 --- a/documentation/src/main/resources/pages/ditto/architecture-services-connectivity.md +++ b/documentation/src/main/resources/pages/ditto/architecture-services-connectivity.md @@ -18,15 +18,15 @@ connectivity service offers a flexible and customizable [payload mapping] on top The model of the connectivity service is defined around the entity `Connection`: -* [model](https://github.com/eclipse/ditto/tree/master/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model) +* [model](https://github.com/eclipse-ditto/ditto/tree/master/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model) ## Signals Other services can communicate with the connectivity service via: -* [ConnectivityCommands](https://github.com/eclipse/ditto/tree/master/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java): +* [ConnectivityCommands](https://github.com/eclipse-ditto/ditto/tree/master/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/commands/ConnectivityCommand.java): implementing classes provide commands which are processed by this service -* [ConnectivityEvents](https://github.com/eclipse/ditto/tree/master/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/events/ConnectivityEvent.java): +* [ConnectivityEvents](https://github.com/eclipse-ditto/ditto/tree/master/connectivity/model/src/main/java/org/eclipse/ditto/connectivity/model/signals/events/ConnectivityEvent.java): implementing classes represent events which are emitted when entities managed by this service were modified ## Persistence diff --git a/documentation/src/main/resources/pages/ditto/architecture-services-policies.md b/documentation/src/main/resources/pages/ditto/architecture-services-policies.md index ab171527f17..e05292e3ca4 100644 --- a/documentation/src/main/resources/pages/ditto/architecture-services-policies.md +++ b/documentation/src/main/resources/pages/ditto/architecture-services-policies.md @@ -12,15 +12,15 @@ The "policies" service takes care of persisting [Policies](basic-policy.html). The model of the policies service is defined around the entity `Policy`: -* [Policy model](https://github.com/eclipse/ditto/tree/master/policies/model/src/main/java/org/eclipse/ditto/policies/model) +* [Policy model](https://github.com/eclipse-ditto/ditto/tree/master/policies/model/src/main/java/org/eclipse/ditto/policies/model) ## Signals Other services can communicate with the policies service via: -* [PolicyCommands](https://github.com/eclipse/ditto/tree/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java): +* [PolicyCommands](https://github.com/eclipse-ditto/ditto/tree/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java): implementing classes provide commands which are processed by this service -* [PolicyEvents](https://github.com/eclipse/ditto/tree/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java): +* [PolicyEvents](https://github.com/eclipse-ditto/ditto/tree/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/events/PolicyEvent.java): implementing classes represent events which are emitted when entities managed by this service were modified ## Persistence diff --git a/documentation/src/main/resources/pages/ditto/architecture-services-things-search.md b/documentation/src/main/resources/pages/ditto/architecture-services-things-search.md index a63c8e9785e..05e64b1f221 100644 --- a/documentation/src/main/resources/pages/ditto/architecture-services-things-search.md +++ b/documentation/src/main/resources/pages/ditto/architecture-services-things-search.md @@ -21,14 +21,14 @@ and [policies](architecture-services-policies.html) services. It however contains a model which can transform an RQL search query into a Java domain model which is defined here: -* [rql parser ast](https://github.com/eclipse/ditto/tree/master/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast) +* [rql parser ast](https://github.com/eclipse-ditto/ditto/tree/master/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast) ## Signals Other services can communicate with the things-search service via: -* [commands](https://github.com/eclipse/ditto/tree/master/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands): +* [commands](https://github.com/eclipse-ditto/ditto/tree/master/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands): containing commands and command responses which are processed by this service ## Persistence diff --git a/documentation/src/main/resources/pages/ditto/architecture-services-things.md b/documentation/src/main/resources/pages/ditto/architecture-services-things.md index 7c8936f2202..0bdebcc0c2e 100644 --- a/documentation/src/main/resources/pages/ditto/architecture-services-things.md +++ b/documentation/src/main/resources/pages/ditto/architecture-services-things.md @@ -11,15 +11,15 @@ The "things" service takes care of persisting [Things](basic-thing.html) and [Fe The model of the things service is defined around the entities `Thing` and `Feature`: -* [Thing model](https://github.com/eclipse/ditto/tree/master/things/model/src/main/java/org/eclipse/ditto/things/model) +* [Thing model](https://github.com/eclipse-ditto/ditto/tree/master/things/model/src/main/java/org/eclipse/ditto/things/model) ## Signals Other services can communicate with the things service via: -* [ThingCommands](https://github.com/eclipse/ditto/tree/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/ThingCommand.java): +* [ThingCommands](https://github.com/eclipse-ditto/ditto/tree/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/ThingCommand.java): implementing classes provide commands which are processed by this service -* [ThingEvents](https://github.com/eclipse/ditto/tree/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/events/ThingEvent.java): +* [ThingEvents](https://github.com/eclipse-ditto/ditto/tree/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/events/ThingEvent.java): implementing classes represent events which are emitted when entities managed by this service were modified ## Persistence diff --git a/documentation/src/main/resources/pages/ditto/basic-namespaces-and-names.md b/documentation/src/main/resources/pages/ditto/basic-namespaces-and-names.md index 4f8ef8a90ee..573b1d64f9f 100644 --- a/documentation/src/main/resources/pages/ditto/basic-namespaces-and-names.md +++ b/documentation/src/main/resources/pages/ditto/basic-namespaces-and-names.md @@ -19,7 +19,7 @@ The namespace must conform to the following notation: When writing a Java application, you can use the following regex to validate your namespaces: ``(?|(?:(?:[a-zA-Z]\w*+)(?:[.-][a-zA-Z]\w*+)*+))`` - (see [RegexPatterns#NAMESPACE_REGEX](https://github.com/eclipse/ditto/blob/master/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/RegexPatterns.java)). + (see [RegexPatterns#NAMESPACE_REGEX](https://github.com/eclipse-ditto/ditto/blob/master/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/RegexPatterns.java)). Examples for valid namespaces: * `org.eclipse.ditto`, @@ -36,7 +36,7 @@ The name must conform to the following notation: When writing a Java application, you can use the following regex to validate your thing name: ``(?[^\x00-\x1F\x7F-\xFF/]++)`` - (see [RegexPatterns#ENTITY_NAME_REGEX](https://github.com/eclipse/ditto/blob/master/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/RegexPatterns.java)). + (see [RegexPatterns#ENTITY_NAME_REGEX](https://github.com/eclipse-ditto/ditto/blob/master/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/RegexPatterns.java)). Examples for valid names: * `ditto`, @@ -53,7 +53,7 @@ A namespaced ID must conform to the following expectations: When writing a Java application, you can use the following regex to validate your namespaced IDs: ``(?|(?:(?:[a-zA-Z]\w*+)(?:[.-][a-zA-Z]\w*+)*+)):(?[^\x00-\x1F\x7F-\xFF/]++)`` - (see [RegexPatterns#ID_REGEX](https://github.com/eclipse/ditto/blob/master/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/RegexPatterns.java)). + (see [RegexPatterns#ID_REGEX](https://github.com/eclipse-ditto/ditto/blob/master/base/model/src/main/java/org/eclipse/ditto/base/model/entity/id/RegexPatterns.java)). Examples for valid IDs: * `org.eclipse.ditto:smart-coffee-1`, diff --git a/documentation/src/main/resources/pages/ditto/basic-wot-integration-example.md b/documentation/src/main/resources/pages/ditto/basic-wot-integration-example.md index 38172367068..2d2ecad67df 100644 --- a/documentation/src/main/resources/pages/ditto/basic-wot-integration-example.md +++ b/documentation/src/main/resources/pages/ditto/basic-wot-integration-example.md @@ -15,7 +15,7 @@ Wrapping up the [WoT integration](basic-wot-integration.html) with a practical e You can provide a WoT Thing Model via any HTTP(s) URL addressable endpoint, for example simply put your WoT TMs into a GitHub repository. For this example, Ditto added a model into its `ditto-examples` GitHub Repo: -[floor-lamp-1.0.0.tm.jsonld](https://github.com/eclipse/ditto-examples/blob/master/wot/models/floor-lamp-1.0.0.tm.jsonld) +[floor-lamp-1.0.0.tm.jsonld](https://github.com/eclipse-ditto/ditto-examples/blob/master/wot/models/floor-lamp-1.0.0.tm.jsonld) This file is available as HTTP served file at: [https://eclipse-ditto.github.io/ditto-examples/wot/models/floor-lamp-1.0.0.tm.jsonld](https://eclipse-ditto.github.io/ditto-examples/wot/models/floor-lamp-1.0.0.tm.jsonld) diff --git a/documentation/src/main/resources/pages/ditto/basic-wot-integration.md b/documentation/src/main/resources/pages/ditto/basic-wot-integration.md index aece030ee24..43ee2ffe879 100644 --- a/documentation/src/main/resources/pages/ditto/basic-wot-integration.md +++ b/documentation/src/main/resources/pages/ditto/basic-wot-integration.md @@ -294,7 +294,7 @@ Prerequisites to use the Thing Description generation: when retrieving the Thing via HTTP `GET /api/2/:` The available configuration of the WoT integration can be found in the -[things.conf](https://github.com/eclipse/ditto/blob/master/things/service/src/main/resources/things.conf) +[things.conf](https://github.com/eclipse-ditto/ditto/blob/master/things/service/src/main/resources/things.conf) config file of the [things service](architecture-services-things.html) at path `ditto.things.wot`. There you can configure which `securityDefinitions` shall be added to the generated TDs and which `base` path prefix to create into the TDs, depending on your public Ditto endpoint. @@ -370,7 +370,7 @@ In order to resolve TM placeholders, Ditto applies the following strategy: * when a placeholder was not found in the `"model-placeholders"` of the Thing/Feature, a fallback to the Ditto configuration is done: * placeholder fallbacks can be configured in Ditto via the - [things.conf](https://github.com/eclipse/ditto/blob/master/things/service/src/main/resources/things.conf) + [things.conf](https://github.com/eclipse-ditto/ditto/blob/master/things/service/src/main/resources/things.conf) configuration file of the [things service](architecture-services-things.html) at path `ditto.things.wot.to-thing-description.placeholders`.
This map may contain static values, but use and JSON type as value (e.g. also a JSON Object), e.g.: diff --git a/documentation/src/main/resources/pages/ditto/client-sdk-java.md b/documentation/src/main/resources/pages/ditto/client-sdk-java.md index 5e2bd1c80f2..9f0e8f82f49 100644 --- a/documentation/src/main/resources/pages/ditto/client-sdk-java.md +++ b/documentation/src/main/resources/pages/ditto/client-sdk-java.md @@ -270,5 +270,5 @@ client.twin().registerForThingChanges("REG1", change -> { ## Further examples For further examples on how to use the Ditto client, please have a look at the class -[DittoClientUsageExamples](https://github.com/eclipse/ditto-clients/blob/master/java/src/test/java/org/eclipse/ditto/client/DittoClientUsageExamples.java) +[DittoClientUsageExamples](https://github.com/eclipse-ditto/ditto-clients/blob/master/java/src/test/java/org/eclipse/ditto/client/DittoClientUsageExamples.java) which is configured to connect to the [Ditto sandbox](https://ditto.eclipseprojects.io). diff --git a/documentation/src/main/resources/pages/ditto/client-sdk-javascript.md b/documentation/src/main/resources/pages/ditto/client-sdk-javascript.md index 27c7cc73e79..be88bf1130c 100644 --- a/documentation/src/main/resources/pages/ditto/client-sdk-javascript.md +++ b/documentation/src/main/resources/pages/ditto/client-sdk-javascript.md @@ -13,13 +13,13 @@ Install `@eclipse-ditto/ditto-javascript-client-dom` for the DOM (browser) imple the API and build your own client implementation. More information can be found in the descriptions of the subpackages: -* [@eclipse-ditto/ditto-javascript-client-api](https://github.com/eclipse/ditto-clients/blob/master/javascript/lib/api/README.md) -* [@eclipse-ditto/ditto-javascript-client-dom](https://github.com/eclipse/ditto-clients/blob/master/javascript/lib/dom/README.md) -* [@eclipse-ditto/ditto-javascript-client-node](https://github.com/eclipse/ditto-clients/blob/master/javascript/lib/node/README.md) +* [@eclipse-ditto/ditto-javascript-client-api](https://github.com/eclipse-ditto/ditto-clients/blob/master/javascript/lib/api/README.md) +* [@eclipse-ditto/ditto-javascript-client-dom](https://github.com/eclipse-ditto/ditto-clients/blob/master/javascript/lib/dom/README.md) +* [@eclipse-ditto/ditto-javascript-client-node](https://github.com/eclipse-ditto/ditto-clients/blob/master/javascript/lib/node/README.md) All released versions are published on [npmjs.com](https://www.npmjs.com/~eclipse_ditto). -## Compatibility with [Eclipse Ditto](https://github.com/eclipse/ditto) +## Compatibility with [Eclipse Ditto](https://github.com/eclipse-ditto/ditto) The newest release of the JavaScript client will always try to cover as much API functionality of the same Eclipse Ditto major version as possible. There might diff --git a/documentation/src/main/resources/pages/ditto/connectivity-hmac-signing.md b/documentation/src/main/resources/pages/ditto/connectivity-hmac-signing.md index 42badf66df5..ad11a5a2d1e 100644 --- a/documentation/src/main/resources/pages/ditto/connectivity-hmac-signing.md +++ b/documentation/src/main/resources/pages/ditto/connectivity-hmac-signing.md @@ -93,7 +93,7 @@ The parameters of the algorithm `az-sasl` are: ### Configuration -Algorithm names and implementations are configured in [`connectivity.conf`](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf). +Algorithm names and implementations are configured in [`connectivity.conf`](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf). The default configuration provides the names and implementations of the available pre-defined algorithms for the given connection types. ```hocon @@ -130,7 +130,7 @@ and which interface needs to be implemented. | | HTTP Push connection | AMQP 1.0 connection | | ------------------------ | -------------------- | ------------------- | | Config path | `ditto.connectivity.connection.http-push.hmac-algorithms` | `ditto.connectivity.connection.amqp10.hmac-algorithms` | -| Class to implement | [HttpRequestSigningFactory](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/httppush/HttpRequestSigningFactory.java) | [AmqpConnectionSigningFactory](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/amqp/AmqpConnectionSigningFactory.java) | +| Class to implement | [HttpRequestSigningFactory](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/httppush/HttpRequestSigningFactory.java) | [AmqpConnectionSigningFactory](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/amqp/AmqpConnectionSigningFactory.java) | ### Example integrations diff --git a/documentation/src/main/resources/pages/ditto/connectivity-manage-connections.md b/documentation/src/main/resources/pages/ditto/connectivity-manage-connections.md index 4e2a595f752..da222f593cb 100644 --- a/documentation/src/main/resources/pages/ditto/connectivity-manage-connections.md +++ b/documentation/src/main/resources/pages/ditto/connectivity-manage-connections.md @@ -226,7 +226,7 @@ The contained fields in a single log entry are the following: * `message`: the actual log message Please inspect the other available configuration options in -[connectivity.conf](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) +[connectivity.conf](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) at path `ditto.connectivity.monitoring.logger.publisher` to learn about other configuration possibilities. diff --git a/documentation/src/main/resources/pages/ditto/connectivity-mapping.md b/documentation/src/main/resources/pages/ditto/connectivity-mapping.md index d03d3af5c7b..edc9989b693 100644 --- a/documentation/src/main/resources/pages/ditto/connectivity-mapping.md +++ b/documentation/src/main/resources/pages/ditto/connectivity-mapping.md @@ -1185,8 +1185,8 @@ Beside the JavaScript based mapping - which can be configured/changed at runtime connectivity service - there is also the possibility to implement a custom Java based mapper. The interface to be implemented is -[`MessageMapper`](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/MessageMapper.java)) -and there is an abstract class [`AbstractMessageMapper`](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/AbstractMessageMapper.java) +[`MessageMapper`](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/MessageMapper.java)) +and there is an abstract class [`AbstractMessageMapper`](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/mapping/AbstractMessageMapper.java) which eases implementation of a custom mapper. Simply extend from `AbstractMessageMapper` to provide a custom mapper: @@ -1278,7 +1278,7 @@ In order to use this custom Java based mapper implementation, the following step ### Example for Custom Java based mapper Please have a look at the following Ditto example project: -* [custom-ditto-java-payload-mapper](https://github.com/eclipse/ditto-examples/tree/master/custom-ditto-java-payload-mapper) +* [custom-ditto-java-payload-mapper](https://github.com/eclipse-ditto/ditto-examples/tree/master/custom-ditto-java-payload-mapper) This shows how to implement, add and configure a custom, Protobuf based, Java payload mapper for Ditto to use in the connectivity service for mapping a custom domain specific Protbuf encoded payload. diff --git a/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-kafka2.md b/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-kafka2.md index 3bbcc50e272..30c67ebe4b2 100644 --- a/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-kafka2.md +++ b/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-kafka2.md @@ -22,7 +22,7 @@ be configured for the connection in order to transform the messages. ## Global Kafka client configuration -The behavior of the used Kafka client can be configured in the [connectivity.conf](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) +The behavior of the used Kafka client can be configured in the [connectivity.conf](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) under key `ditto.connectivity.connection.kafka`: * `consumer`: The Kafka consumer configuration applied when configuring [sources](#source-format) in order to consume messages from Kafka * `committer`: The Kafka committer configuration to apply when consuming messages, e.g. the `max-batch` size and `max-interval` duration diff --git a/documentation/src/main/resources/pages/ditto/feedback.md b/documentation/src/main/resources/pages/ditto/feedback.md index b9222adea96..f075ba5bc6f 100644 --- a/documentation/src/main/resources/pages/ditto/feedback.md +++ b/documentation/src/main/resources/pages/ditto/feedback.md @@ -9,6 +9,6 @@ topnav: topnav You have following possibilities in order to get support or give feedback: * Join the chat at [https://gitter.im/eclipse/ditto](https://gitter.im/eclipse/ditto?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) for questions -* start a [GitHub discussion](https://github.com/eclipse/ditto/discussions) for questions and general discussions +* start a [GitHub discussion](https://github.com/eclipse-ditto/ditto/discussions) for questions and general discussions * As your question on [StackOverflow](https://stackoverflow.com/questions/tagged/eclipse-ditto) -* create a [GitHub issue](https://github.com/eclipse/ditto/issues) for found bugs / feature requests +* create a [GitHub issue](https://github.com/eclipse-ditto/ditto/issues) for found bugs / feature requests diff --git a/documentation/src/main/resources/pages/ditto/installation-extending.md b/documentation/src/main/resources/pages/ditto/installation-extending.md index 1bf0a83cadc..739764ae882 100644 --- a/documentation/src/main/resources/pages/ditto/installation-extending.md +++ b/documentation/src/main/resources/pages/ditto/installation-extending.md @@ -65,7 +65,7 @@ If however lots of configuration changes should be done, a more feasible approac Those configuration files can contain any [Ditto configuration](installation-operating.html#ditto-configuration) done in the service config files. -For example, the [gateway.conf](https://github.com/eclipse/ditto/blob/master/gateway/service/src/main/resources/gateway.conf) +For example, the [gateway.conf](https://github.com/eclipse-ditto/ditto/blob/master/gateway/service/src/main/resources/gateway.conf) contains the following configuration snippet: ```hocon ditto { diff --git a/documentation/src/main/resources/pages/ditto/installation-operating.md b/documentation/src/main/resources/pages/ditto/installation-operating.md index aa90da3fbaf..0685bde2b5e 100644 --- a/documentation/src/main/resources/pages/ditto/installation-operating.md +++ b/documentation/src/main/resources/pages/ditto/installation-operating.md @@ -35,11 +35,11 @@ Each of Ditto's microservice has many options for configuration, e.g. timeouts, In order to have a look at all possible configuration options and what default values they have, here are the configuration files of Ditto's microservices: -* Policies: [policies.conf](https://github.com/eclipse/ditto/blob/master/policies/service/src/main/resources/policies.conf) -* Things: [things.conf](https://github.com/eclipse/ditto/blob/master/things/service/src/main/resources/things.conf) -* Things-Search: [things-search.conf](https://github.com/eclipse/ditto/blob/master/thingsearch/service/src/main/resources/things-search.conf) -* Connectivity: [connectivity.conf](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) -* Gateway: [gateway.conf](https://github.com/eclipse/ditto/blob/master/gateway/service/src/main/resources/gateway.conf) +* Policies: [policies.conf](https://github.com/eclipse-ditto/ditto/blob/master/policies/service/src/main/resources/policies.conf) +* Things: [things.conf](https://github.com/eclipse-ditto/ditto/blob/master/things/service/src/main/resources/things.conf) +* Things-Search: [things-search.conf](https://github.com/eclipse-ditto/ditto/blob/master/thingsearch/service/src/main/resources/things-search.conf) +* Connectivity: [connectivity.conf](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) +* Gateway: [gateway.conf](https://github.com/eclipse-ditto/ditto/blob/master/gateway/service/src/main/resources/gateway.conf) Whenever you find the syntax `${?UPPER_CASE_ENV_NAME}` in the configuration files, you may overwrite the default value by specifying that environment variable when running the container. @@ -232,7 +232,7 @@ Values that are valid URIs are treated specially and only the password of the us encrypted. Configuration can be seen at [Ditto service configuration files](#ditto-configuration) in -the [connectivity.conf](https://github.com/eclipse/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) +the [connectivity.conf](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) at "ditto.connectivity.connection.encryption" section of the config. If at some point encryption is decided to be disabled the symmetric key is important to be kept in the @@ -412,7 +412,7 @@ To put it in a nutshell, Ditto reports: * mapping times Have a look at the -[example Grafana dashboards](https://github.com/eclipse/ditto/tree/master/deployment/operations/grafana-dashboards) +[example Grafana dashboards](https://github.com/eclipse-ditto/ditto/tree/master/deployment/operations/grafana-dashboards) and build and share new ones back to the Ditto community. ## Tracing @@ -690,14 +690,14 @@ Piggyback commands can be used for managing policies, e.g. in order to create, r "devops" (super) user. All -[PolicyCommand](https://github.com/eclipse/ditto/blob/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java)s +[PolicyCommand](https://github.com/eclipse-ditto/ditto/blob/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/PolicyCommand.java)s may be sent via piggyback - however be aware that the internal JSON representation of the policy commands must be used and not the [Ditto Protocol](protocol-specification-policies.html). The internal JSON representation can be found in the code, e.g. defined in the static `fromJson` methods of the commands. Example piggyback for -[CreatePolicy](https://github.com/eclipse/ditto/blob/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/modify/CreatePolicy.java): +[CreatePolicy](https://github.com/eclipse-ditto/ditto/blob/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/modify/CreatePolicy.java): ```json { "targetActorSelection": "/system/sharding/policy", @@ -719,7 +719,7 @@ Example piggyback for ``` Example piggyback for -[RetrievePolicy](https://github.com/eclipse/ditto/blob/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java): +[RetrievePolicy](https://github.com/eclipse-ditto/ditto/blob/master/policies/model/src/main/java/org/eclipse/ditto/policies/model/signals/commands/query/RetrievePolicy.java): ```json { "targetActorSelection": "/system/sharding/policy", @@ -741,14 +741,14 @@ Piggyback commands can be used for managing things, e.g. in order to create, ret "devops" (super) user. All -[ThingCommand](https://github.com/eclipse/ditto/blob/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/ThingCommand.java)s +[ThingCommand](https://github.com/eclipse-ditto/ditto/blob/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/ThingCommand.java)s may be sent via piggyback - however be aware that the internal JSON representation of the thing commands must be used and not the [Ditto Protocol](protocol-specification-things.html). The internal JSON representation can be found in the code, e.g. defined in the static `fromJson` methods of the commands. Example piggyback for -[CreateThing](https://github.com/eclipse/ditto/blob/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/CreateThing.java): +[CreateThing](https://github.com/eclipse-ditto/ditto/blob/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/modify/CreateThing.java): ```json { "targetActorSelection": "/system/sharding/thing", @@ -768,7 +768,7 @@ Example piggyback for ``` Example piggyback for -[RetrieveThing](https://github.com/eclipse/ditto/blob/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/query/RetrieveThing.java): +[RetrieveThing](https://github.com/eclipse-ditto/ditto/blob/master/things/model/src/main/java/org/eclipse/ditto/things/model/signals/commands/query/RetrieveThing.java): ```json { "targetActorSelection": "/system/sharding/thing", diff --git a/documentation/src/main/resources/pages/ditto/installation-running.md b/documentation/src/main/resources/pages/ditto/installation-running.md index e2aa2005f20..231d699e245 100644 --- a/documentation/src/main/resources/pages/ditto/installation-running.md +++ b/documentation/src/main/resources/pages/ditto/installation-running.md @@ -26,7 +26,7 @@ In order to start Ditto, you'll need: * some other tools like docker-compose, helm, k3s, minikube or openshift to run Ditto. You can choose from several options to run/deploy Ditto. -A good starting point here is [Ditto Deployment](https://github.com/eclipse/ditto/blob/master/deployment/README.md). +A good starting point here is [Ditto Deployment](https://github.com/eclipse-ditto/ditto/blob/master/deployment/README.md). After completing the deployment of your choice Ditto should be up & running. Now you have running: diff --git a/documentation/src/main/resources/pages/ditto/intro-hello-world.md b/documentation/src/main/resources/pages/ditto/intro-hello-world.md index 61283b69bac..32d53683fa2 100644 --- a/documentation/src/main/resources/pages/ditto/intro-hello-world.md +++ b/documentation/src/main/resources/pages/ditto/intro-hello-world.md @@ -62,7 +62,7 @@ Inside "definition" we can add one JSON string value. We create a Thing for the example from above by using [cURL](https://github.com/curl/curl). Basic authentication will use the credentials of a user "ditto". Those credentials have been created by default in the [nginx](https://github.com/nginx/nginx) started via "docker". -(See [ditto/deployment/docker/README.md](https://github.com/eclipse/ditto/blob/master/deployment/docker/README.md)) +(See [ditto/deployment/docker/README.md](https://github.com/eclipse-ditto/ditto/blob/master/deployment/docker/README.md)) ```bash curl -u ditto:ditto -X PUT -H 'Content-Type: application/json' -d '{ diff --git a/documentation/src/main/resources/pages/ditto/presentations.md b/documentation/src/main/resources/pages/ditto/presentations.md index 7d6b106c737..f68e7a09618 100644 --- a/documentation/src/main/resources/pages/ditto/presentations.md +++ b/documentation/src/main/resources/pages/ditto/presentations.md @@ -88,7 +88,7 @@ Organized via [Meetup](https://www.meetup.com/IoT-Hessen/events/248886802/) by D The slides can be found here: [2018_05_23-meetup-iot-hessen](slides/2018_05_23-meetup-iot-hessen/index.html). -The code of the showed live demo can be found in our [ditto-examples](https://github.com/eclipse/ditto-examples/tree/master/octopus-via-hono). +The code of the showed live demo can be found in our [ditto-examples](https://github.com/eclipse-ditto/ditto-examples/tree/master/octopus-via-hono). ## 07.02.2018 Virtual IoT Meetup diff --git a/documentation/src/main/resources/pages/ditto/release_notes_010M3.md b/documentation/src/main/resources/pages/ditto/release_notes_010M3.md index faf6238293b..44d576175e5 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_010M3.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_010M3.md @@ -15,7 +15,7 @@ bugfixes were added. ## New features -### [AMQP Bridge](https://github.com/eclipse/ditto/pull/65) +### [AMQP Bridge](https://github.com/eclipse-ditto/ditto/pull/65) A new service "ditto-amqp-bridge" was added in order to be able establish a connection to AMQP 1.0 endpoints like for example [Eclipse Hono](https://eclipse.org/hono/). @@ -24,7 +24,7 @@ That way messages in [Ditto Protocol](protocol-overview.html) coming from Eclips For more details, please have a look at the [AMQP 1.0 binding](connectivity-protocol-bindings-amqp10.html) and the [AMQP-Bridge architecture](architecture-services-connectivity.html). -### [DevOps commands HTTP endpoint](https://github.com/eclipse/ditto/pull/55) +### [DevOps commands HTTP endpoint](https://github.com/eclipse-ditto/ditto/pull/55) In order to dynamically make changes to a running Ditto cluster without restarting, Ditto added an implementation of so called "DevOps commands". Those can be triggered via a HTTP API for all services at once or specifically targeted @@ -39,7 +39,7 @@ Further information can be found in the [operating chapter](installation-operati ## Bugfixes -### [Stabilization of eventually consistent search index](https://github.com/eclipse/ditto/pull/83) +### [Stabilization of eventually consistent search index](https://github.com/eclipse-ditto/ditto/pull/83) In various conditions the search index which is updated by the [search](basic-search.html) of Ditto was not updated in case events were missed or there were timing issues. @@ -48,7 +48,7 @@ Those issues were resolved by improving the approach for keeping track of the al ### Various smaller bugfixes -This is a complete list of the [merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.1.0-M3+). +This is a complete list of the [merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.1.0-M3+). ## Documentation diff --git a/documentation/src/main/resources/pages/ditto/release_notes_020M1.md b/documentation/src/main/resources/pages/ditto/release_notes_020M1.md index 3b710b11f0e..1d1c676e3bc 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_020M1.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_020M1.md @@ -12,7 +12,7 @@ bugfixes were added. ## New features -### [Search in namespaces](https://github.com/eclipse/ditto/pull/104) +### [Search in namespaces](https://github.com/eclipse-ditto/ditto/pull/104) A query parameter `namespaces` was added to the [HTTP search API](httpapi-search.html). It can be used in order to restrict search to Things within specific namespaces. For example, with the route @@ -25,7 +25,7 @@ only Things with IDs of the form `john:` and `mark:` are r Namespace restriction happens at the start of a search query execution and may speed up a search queries considerably. -### [Feature Definition](https://github.com/eclipse/ditto/issues/60) +### [Feature Definition](https://github.com/eclipse-ditto/ditto/issues/60) Ditto's model (to be precise the `Feature`) was enhanced by a `Definition`. This field is intended to store which contract a Feature follows (which state and capabilities can be expected from a Feature). @@ -38,14 +38,14 @@ a look at [its documentation](basic-feature.html#feature-definition). ## Bugfixes -### [AMQP 1.0 failover is not working](https://github.com/eclipse/ditto/issues/97) +### [AMQP 1.0 failover is not working](https://github.com/eclipse-ditto/ditto/issues/97) Using `"failover": true` when creating a new AMQP 1.0 connection caused that the connection could not be established. ### Various smaller bugfixes -This is a complete list of the [merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.2.0-M1+). +This is a complete list of the [merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.2.0-M1+). ## Documentation diff --git a/documentation/src/main/resources/pages/ditto/release_notes_030M1.md b/documentation/src/main/resources/pages/ditto/release_notes_030M1.md index c5785a09b3d..0f39893317f 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_030M1.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_030M1.md @@ -12,7 +12,7 @@ bugfixes were added. ## Changes -### [OpenJ9 based Docker images](https://github.com/eclipse/ditto/pull/133) +### [OpenJ9 based Docker images](https://github.com/eclipse-ditto/ditto/pull/133) The official Eclipse Ditto Docker images are now based on the [Eclipse OpenJ9](https://www.eclipse.org/openj9/) JVM. With this JVM (cheers to the OpenJ9 developers for this awesome JVM) Ditto's containers need a lot less memory having @@ -28,12 +28,12 @@ the [new features](#new-features). ## New features -### [AMQP 0.9.1 connectivity](https://github.com/eclipse/ditto/issues/129) +### [AMQP 0.9.1 connectivity](https://github.com/eclipse-ditto/ditto/issues/129) The new `connectivity` service can now, additionally to AMQP 1.0, manage and open connections to AMQP 0.9.1 endpoints (e.g. provided by a [RabbitMQ](https://www.rabbitmq.com) broker). -### [Payload mapping to/from Ditto Protocol](https://github.com/eclipse/ditto/issues/130) +### [Payload mapping to/from Ditto Protocol](https://github.com/eclipse-ditto/ditto/issues/130) The new `connectivity` service can now also map message arbitrary text or byte payload from incoming AMQP 1.0 / 0.9.1 connections which are not yet in [Ditto Protocol](protocol-overview.html) messages in such and can also map outgoing @@ -47,7 +47,7 @@ Ditto Protocol messages (e.g. responses or events) back to some arbitrary text o The former AMQP bridge did loose the connection to AMQP 1.0 endpoints. This is now much more stable from which also the new AMQP 0.9.1 connections benefit. -### [Docker compose config was wrong](https://github.com/eclipse/ditto/issues/140) +### [Docker compose config was wrong](https://github.com/eclipse-ditto/ditto/issues/140) The entrypoint/command was pointing to a wrong `starter.jar`. @@ -55,7 +55,7 @@ The entrypoint/command was pointing to a wrong `starter.jar`. ### Various smaller bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.3.0-M1+). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.3.0-M1+). ## Documentation diff --git a/documentation/src/main/resources/pages/ditto/release_notes_030M2.md b/documentation/src/main/resources/pages/ditto/release_notes_030M2.md index cbdc8273f81..f94dbc8f471 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_030M2.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_030M2.md @@ -12,7 +12,7 @@ bugfixes were added. ## Changes -### [Reduce network load for cache-sync](https://github.com/eclipse/ditto/issues/126) +### [Reduce network load for cache-sync](https://github.com/eclipse-ditto/ditto/issues/126) With 0.3.0-M1 Ditto had a performance issue when managing more than ~100.000 Things in its memory as Ditto used a distributed cluster cache which was not intended to be used in that way. Over time, as cache entries could not be deleted @@ -25,7 +25,7 @@ This is the biggest change in this milestone and required a lot of refactoring e committers Daniel and Yufei who did an amazing job: the roundtrip times in a Ditto cluster are now at a constant and very good rate. -### [Cluster bootstrapping improved](https://github.com/eclipse/ditto/issues/167) +### [Cluster bootstrapping improved](https://github.com/eclipse-ditto/ditto/issues/167) Ditto now uses the [akka-management](https://developer.lightbend.com/docs/akka-management/current/index.html) library in order to bootstrap a new cluster. By default Ditto now uses a DNS-based approach to find its other cluster-nodes and @@ -49,13 +49,13 @@ The search-index of the [Ditto search](basic-search.html) had several issues whi when searching for Things. These issues were adressed in several fixes: -* [#159](https://github.com/eclipse/ditto/pull/159) -* [#169](https://github.com/eclipse/ditto/pull/169) -* [#175](https://github.com/eclipse/ditto/pull/175) +* [#159](https://github.com/eclipse-ditto/ditto/pull/159) +* [#169](https://github.com/eclipse-ditto/ditto/pull/169) +* [#175](https://github.com/eclipse-ditto/ditto/pull/175) ### Various smaller bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.3.0-M2+). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.3.0-M2+). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_080.md b/documentation/src/main/resources/pages/ditto/release_notes_080.md index 9cdc7291ee8..f9b3e0fa386 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_080.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_080.md @@ -41,7 +41,7 @@ bugfixes were added. #### New features -##### [Support Hono's command&control in Ditto connectivity](https://github.com/eclipse/ditto/issues/164) +##### [Support Hono's command&control in Ditto connectivity](https://github.com/eclipse-ditto/ditto/issues/164) Eclipse Ditto can now map arbitrary headers when connecting to AMQP 1.0 endpoints or AMQP 0.9.1 brokers. That way Ditto can send "command&control" messages to Eclipse Hono and correlate a potential response coming from a @@ -50,7 +50,7 @@ device. #### Bugfixes This release contains several bugfixes, this is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.8.0+). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.8.0+). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_080M1.md b/documentation/src/main/resources/pages/ditto/release_notes_080M1.md index 62fcfd36090..3163955acac 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_080M1.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_080M1.md @@ -22,13 +22,13 @@ bugfixes were added. ## Changes -### [Marked some DittoHeaders as internal](https://github.com/eclipse/ditto/pull/195) +### [Marked some DittoHeaders as internal](https://github.com/eclipse-ditto/ditto/pull/195) In order to prevent that a user of Ditto's API (e.g. WebSocket or AMQP) sets arbitrary security relevant headers, those `DittoHeaders` are no marked as "not readable from external". Other headers which should not be propagated to the outside of Ditto are marked as "not to be written to external". -### [Update to Kamon 1.0 and report metrics to Prometheus](https://github.com/eclipse/ditto/issues/105) +### [Update to Kamon 1.0 and report metrics to Prometheus](https://github.com/eclipse-ditto/ditto/issues/105) Previously, Ditto was using Kamon 0.6.x and reported metrics/traces to a [Graphite](https://graphiteapp.org) back-end. Together with the update to Kamon 1.0 the Ditto services now provide HTTP endpoints which can be scraped by @@ -40,14 +40,14 @@ guide. ## New features -### [Kubernetes cluster bootstrapping](https://github.com/eclipse/ditto/pull/201) +### [Kubernetes cluster bootstrapping](https://github.com/eclipse-ditto/ditto/pull/201) Ditto now discovers its cluster nodes automatically also when running in Kubernetes. It uses the kubernetes-api in order to discover the other cluster nodes. An example on how to run Ditto with Kubernetes is provided in the /kubernetes git directory. -### [Allow placeholders inside connection config](https://github.com/eclipse/ditto/issues/178) +### [Allow placeholders inside connection config](https://github.com/eclipse-ditto/ditto/issues/178) In many cases configuration values of a [Connection](connectivity-manage-connections.html) can be dependent on the message's headers. This feature allows Ditto connections to access header fields and use it in the connection's @@ -59,7 +59,7 @@ from Hono authenticated `device-id` via the placeholder `header:device-id`. ## Bugfixes -### [Fixed excessive memory consumption of things-service](https://github.com/eclipse/ditto/pull/194) +### [Fixed excessive memory consumption of things-service](https://github.com/eclipse-ditto/ditto/pull/194) In previous versions, Ditto's `things-service` created a lot of instances for each Thing which was loaded into memory. This is now optimized so that the service does no longer need so much memory. @@ -71,11 +71,11 @@ In this milestone we put a lot of effort in further stabilizing AMQP 1.0 and 0.9 These issues were addressed in several fixes: -* [#189](https://github.com/eclipse/ditto/pull/189) -* [#178](https://github.com/eclipse/ditto/issues/178) +* [#189](https://github.com/eclipse-ditto/ditto/pull/189) +* [#178](https://github.com/eclipse-ditto/ditto/issues/178) ### Various smaller bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.8.0-M1+). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.8.0-M1+). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_080M2.md b/documentation/src/main/resources/pages/ditto/release_notes_080M2.md index 28631cca3d7..e49c2f3ec84 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_080M2.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_080M2.md @@ -15,7 +15,7 @@ bugfixes were added. ## Changes -### [Define and enforce max. entity sizes in Ditto cluster](https://github.com/eclipse/ditto/issues/221) +### [Define and enforce max. entity sizes in Ditto cluster](https://github.com/eclipse-ditto/ditto/issues/221) In previous versions of Ditto the entity sizes (e.g. for things and policies) were technically not limited. The only implicit limit was a max. cluster message size of 10m. As Ditto is not intended to manage digital twins which are that @@ -25,29 +25,29 @@ big, the sizes of the twins are now by default limited to: * max [Message payload](basic-messages.html) size: 250k They can be adjusted to other needs via the configuration located in -[ditto-limits.conf](https://github.com/eclipse/ditto/blob/master/services/base/src/main/resources/ditto-limits.conf). +[ditto-limits.conf](https://github.com/eclipse-ditto/ditto/blob/master/services/base/src/main/resources/ditto-limits.conf). The default maximum frame size in the ditto cluster was changed to 256k and can be adjusted in the -[ditto-akka-config.conf](https://github.com/eclipse/ditto/blob/master/services/base/src/main/resources/ditto-akka-config.conf#L93). +[ditto-akka-config.conf](https://github.com/eclipse-ditto/ditto/blob/master/services/base/src/main/resources/ditto-akka-config.conf#L93). ## New features -### [MQTT support](https://github.com/eclipse/ditto/issues/220) +### [MQTT support](https://github.com/eclipse-ditto/ditto/issues/220) -In two big PRs ([#225](https://github.com/eclipse/ditto/pull/225) and [#235](https://github.com/eclipse/ditto/pull/235)) +In two big PRs ([#225](https://github.com/eclipse-ditto/ditto/pull/225) and [#235](https://github.com/eclipse-ditto/ditto/pull/235)) Ditto added support for connecting to MQTT brokers (like for example [Eclipse Mosquitto](https://mosquitto.org)) via its [connectivity feature](connectivity-overview.html). Have a look at the [MQTT protocol binding](connectivity-protocol-bindings-mqtt.html) for details. -### [Subscribing for change notifications by optional filter](https://github.com/eclipse/ditto/issues/149) +### [Subscribing for change notifications by optional filter](https://github.com/eclipse-ditto/ditto/issues/149) Until previous versions, when [subscribing for changes](basic-changenotifications.html), the subscriber always got all changes he was entitled to see (based on [access control](basic-auth.html)). Now it is possible to specify for which changes to subscribe based on the optional `namespaces` to consider and an optional [RQL](basic-rql.html) `filter`. The new feature is documented as [change notification filters](basic-changenotifications.html#filtering). -### [Support for conditional requests](https://github.com/eclipse/ditto/pull/226) +### [Support for conditional requests](https://github.com/eclipse-ditto/ditto/pull/226) Ditto's APIs now support `If-Match` and `If-None-Match` headers specified in [rfc7232](https://tools.ietf.org/html/rfc7232#section-3.2) for `things` and `policies` resources. Have a look at the @@ -57,11 +57,11 @@ can help with advanced interaction-patterns with your twins. ## Bugfixes -### [Reconnection to AMQP 0.9.1 connections](https://github.com/eclipse/ditto/issues/228) +### [Reconnection to AMQP 0.9.1 connections](https://github.com/eclipse-ditto/ditto/issues/228) Reconnection did not always work, e.g. when the AMQP 0.9.1 broker was not reachable for a while. ### Various smaller bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.8.0-M2+). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.8.0-M2+). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_080M3.md b/documentation/src/main/resources/pages/ditto/release_notes_080M3.md index 12698802c09..1072887a84c 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_080M3.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_080M3.md @@ -16,7 +16,7 @@ bugfixes were added. With more and more Things, the search service was slowing down massively. -Two Pull Requests ([#275](https://github.com/eclipse/ditto/pull/275), [#278](https://github.com/eclipse/ditto/pull/278)) +Two Pull Requests ([#275](https://github.com/eclipse-ditto/ditto/pull/275), [#278](https://github.com/eclipse-ditto/ditto/pull/278)) addressed this issue with the following changes: * add an index on `_policyId` and `__policyRev` for the `thingEntities` collection. * add the field `_thingId` to new documents in `policiesBasedSearchIndex`. @@ -30,7 +30,7 @@ addressed this issue with the following changes: with the Ditto cluster stopped) using the {% include file.html title="MongoDB migration script from 0.8.0-M2 to 0.8.0-M3" file="migration_mongodb_0.8.0-M2_0.8.0-M3.js" %}.** -### [Netty 3 was removed from dependencies](https://github.com/eclipse/ditto/issues/161) +### [Netty 3 was removed from dependencies](https://github.com/eclipse-ditto/ditto/issues/161) Due to licensing issues with Netty 3, it was removed in this release and replaced with [Akka's Artery](https://doc.akka.io/docs/akka/current/remoting-artery.html) remoting which uses by default a plain TCP @@ -42,7 +42,7 @@ cluster with all services running the new version. ## New features -### [Apply enforcement for incoming messages in connectivity service](https://github.com/eclipse/ditto/issues/265) +### [Apply enforcement for incoming messages in connectivity service](https://github.com/eclipse-ditto/ditto/issues/265) When adding a [connection](connectivity-manage-connections.html), an optional enforcement (e.g. for [AMQP 1.0](connectivity-protocol-bindings-amqp10.html)) may be configured in order to only accept messages having, @@ -51,7 +51,7 @@ for example, a defined header value. This is also very useful to be used for connecting to [Eclipse Hono](https://eclipse.org/hono/) which sends a header `device_id` in every message which Ditto can check against the ID of the addressed twin. -### [Allow to create a new thing that uses a copied policy](https://github.com/eclipse/ditto/issues/268) +### [Allow to create a new thing that uses a copied policy](https://github.com/eclipse-ditto/ditto/issues/268) When [creating a new Thing](protocol-specification-things-create-or-modify.html) it is now possible to copy the [Policy](basic-policy.html) already used in another Thing. @@ -66,5 +66,5 @@ This milestone contains several bugfixes related to memory leaks, recovery of co ### Various smaller bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.8.0-M3+). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.8.0-M3+). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_090.md b/documentation/src/main/resources/pages/ditto/release_notes_090.md index 39c5e4281e3..e14eada9339 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_090.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_090.md @@ -42,7 +42,7 @@ bugfixes were added. #### Changes -##### [Streamline configuration](https://github.com/eclipse/ditto/issues/350) +##### [Streamline configuration](https://github.com/eclipse-ditto/ditto/issues/350) In this release, we changed how internally the configuration is determined in Ditto's microservices. This should not have any impact on a user of Eclipse Ditto. As however some configuration keys were remanded or restructured, some @@ -51,18 +51,18 @@ adjustments when manually configuring Ditto could be required. #### New features -##### [Introduce cursor-based paging for search requests](https://github.com/eclipse/ditto/pull/407) +##### [Introduce cursor-based paging for search requests](https://github.com/eclipse-ditto/ditto/pull/407) In order to provide a constant performance for using pagination for the `things-search` miscroservice across even large result sets (100s of thousands or more) of digital twins, a new cursor-based approach was added in addition to the old `offset/limit` approach which gets slower for each page. -##### [Simple throttling for amqp 1.0 consumers](https://github.com/eclipse/ditto/pull/420) +##### [Simple throttling for amqp 1.0 consumers](https://github.com/eclipse-ditto/ditto/pull/420) Added a simple throttling mechanism for amqp 1.0 consumers. The throttling is configurable by defining the number of messages allowed per time interval. -##### [Collect connectivity log entries and provide via devops command](https://github.com/eclipse/ditto/issues/318) +##### [Collect connectivity log entries and provide via devops command](https://github.com/eclipse-ditto/ditto/issues/318) In order to let a user analyze failures / unexpected behavior in his connections (e.g. to an MQTT or AMQP broker) without giving him full access to the log-files of Ditto, a connection scoped mechanism for retrieving connectivity logs @@ -71,21 +71,21 @@ was added. #### Bugfixes -##### [Memory and performance fixes in concierge, gateway](https://github.com/eclipse/ditto/pull/400) +##### [Memory and performance fixes in concierge, gateway](https://github.com/eclipse-ditto/ditto/pull/400) Ditto's `concierge` service did use a sharding approach for enforcing authorization information which lead to huge amounts of memory required when managing multi million of digital twins. That, in addition to some memory fixes in the gateway and addition of metrics, was fixed in this PR. -##### [Ensure ordering for processed commands](https://github.com/eclipse/ditto/pull/417) +##### [Ensure ordering for processed commands](https://github.com/eclipse-ditto/ditto/pull/417) In previous versions of Ditto the order in which command were processed not guaranteed to be maintained. As maintaining the order however is the expected behavior, we decided to treat that as bug and added to the `concierge` service (where the order could be lost) to sequentially process commands issues via HTTP/WebSocket and connections. This release contains several bugfixes, this is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.9.0+). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.9.0+). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_090M1.md b/documentation/src/main/resources/pages/ditto/release_notes_090M1.md index a1afe23ff11..e755058ea7b 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_090M1.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_090M1.md @@ -12,32 +12,32 @@ a preview of what to expect in the next release. ## Changes -### [Optimized/reduced memory consumption of ditto-json](https://github.com/eclipse/ditto/pull/304) +### [Optimized/reduced memory consumption of ditto-json](https://github.com/eclipse-ditto/ditto/pull/304) `ditto-json` used quite a lot of memory when building up JSON structures - this change optimized that by a factor of 10-20, so now it is possible to keep ~1.000.000 things in memory with ~4GB of memory. -### [Update several dependencies](https://github.com/eclipse/ditto/issues/300) +### [Update several dependencies](https://github.com/eclipse-ditto/ditto/issues/300) Several used dependencies were updated to their latest (bugfix) releases or even to a stable 1.x version. ## New features -### [Erasing data without downtime](https://github.com/eclipse/ditto/issues/234) +### [Erasing data without downtime](https://github.com/eclipse-ditto/ditto/issues/234) GDPR requires erasure of data on request of data subject. -### [Enhance placeholders by functions](https://github.com/eclipse/ditto/issues/337) +### [Enhance placeholders by functions](https://github.com/eclipse-ditto/ditto/issues/337) The already existing placeholder mechanism for connections was enhanced by optionally adding function calls (currently simple string manipulations like `fn:substring-before(':')`). -### [Connect to Apache Kafka](https://github.com/eclipse/ditto/issues/224) +### [Connect to Apache Kafka](https://github.com/eclipse-ditto/ditto/issues/224) In order to publish Ditto protocol messages (e.g. events/responses/messages/...) to Apache Kafka topics. -### [Collect and provide connection metrics](https://github.com/eclipse/ditto/issues/317) +### [Collect and provide connection metrics](https://github.com/eclipse-ditto/ditto/issues/317) Provide metrics about established connections (e.g. to AMQP 1.0, Kafka, MQTT, ...). @@ -47,5 +47,5 @@ Provide metrics about established connections (e.g. to AMQP 1.0, Kafka, MQTT, .. ### Various smaller bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.9.0-M1+). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.9.0-M1+). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_090M2.md b/documentation/src/main/resources/pages/ditto/release_notes_090M2.md index 67d57c3d214..d27985ce662 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_090M2.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_090M2.md @@ -11,12 +11,12 @@ This second milestone of the "0.9.0" release provides closes more gaps towards t ## Changes -### [Make it easy to add new Authentication mechanisms and allow chaining them](https://github.com/eclipse/ditto/issues/348) +### [Make it easy to add new Authentication mechanisms and allow chaining them](https://github.com/eclipse-ditto/ditto/issues/348) If we have multiple applicable authentication mechanisms all of them are processed until either one of them authenticates successfully or all of them fail. -### [Unify implementations of things-search across API versions](https://github.com/eclipse/ditto/pull/392) +### [Unify implementations of things-search across API versions](https://github.com/eclipse-ditto/ditto/pull/392) Ditto's "search" capabilities were rewritten so that bot API v1 and API v2 use the same MongoDB based search index. @@ -29,24 +29,24 @@ Ditto's "search" capabilities were rewritten so that bot API v1 and API v2 use t ### Various contributions for a setup of Eclipse Ditto on Microsoft Azure -As discussed in the ongoing issue [Eclipse Ditto on Microsoft Azure](https://github.com/eclipse/ditto/issues/358) Microsoft +As discussed in the ongoing issue [Eclipse Ditto on Microsoft Azure](https://github.com/eclipse-ditto/ditto/issues/358) Microsoft added a few PRs in order to deploy Ditto on MS Azure cloud: -* [Enable support for MongoDB persistence in K8s](https://github.com/eclipse/ditto/pull/364) -* [Fix Nginx connectivity after Helm update](https://github.com/eclipse/ditto/pull/375) -* [Recover from closed JMS AMQP message producer](https://github.com/eclipse/ditto/pull/367) -* [Improve nginx configuration for gateway restarts](https://github.com/eclipse/ditto/pull/386) +* [Enable support for MongoDB persistence in K8s](https://github.com/eclipse-ditto/ditto/pull/364) +* [Fix Nginx connectivity after Helm update](https://github.com/eclipse-ditto/ditto/pull/375) +* [Recover from closed JMS AMQP message producer](https://github.com/eclipse-ditto/ditto/pull/367) +* [Improve nginx configuration for gateway restarts](https://github.com/eclipse-ditto/ditto/pull/386) ## Bugfixes -### [MQTT publish fails if no sources are configured](https://github.com/eclipse/ditto/issues/387) +### [MQTT publish fails if no sources are configured](https://github.com/eclipse-ditto/ditto/issues/387) Fixed: If using a MQTT connection that only has targets but no sources, publishing of events will fail. ### Various smaller bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A0.9.0-M2+). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A0.9.0-M2+). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_100.md b/documentation/src/main/resources/pages/ditto/release_notes_100.md index d370a66589c..575e10c463a 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_100.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_100.md @@ -31,7 +31,7 @@ bugfixes were added. ### Changes -#### [Remove suffixed collections](https://github.com/eclipse/ditto/issues/537) +#### [Remove suffixed collections](https://github.com/eclipse-ditto/ditto/issues/537) We removed suffixed collection support from Things and Policies persistence. These collections do not scale well with increased amount of namespaces and lead to massive problems with mongodb as @@ -40,7 +40,7 @@ sharding can't be used. ### New features -#### [Comprehensive support for command responses](https://github.com/eclipse/ditto/issues/540) +#### [Comprehensive support for command responses](https://github.com/eclipse-ditto/ditto/issues/540) Adds the possibility to define a "reply target" for [connection sources](basic-connections.html#sources) where * the response address may be configured @@ -52,7 +52,7 @@ specified a `reply-to` address. Used in combination with [Eclipse Hono](https://eclipse.org/hono/) it is possible to send responses to devices which e.g. need to retrieve data from Ditto. -#### [Add "definition" to Thing in order to reference used model](https://github.com/eclipse/ditto/issues/247) +#### [Add "definition" to Thing in order to reference used model](https://github.com/eclipse-ditto/ditto/issues/247) In order to specify which model a Thing follows, the JSON of the Thing entity was enhanced with a single string for `"definintion"`. This can e.g. be used in order to place an [Eclipse Vorto](https://eclipse.org/vorto/) @@ -60,7 +60,7 @@ In order to specify which model a Thing follows, the JSON of the Thing entity wa ### Bugfixes -#### [Fixed NullPointer in StreamingSessionActor](https://github.com/eclipse/ditto/pull/546) +#### [Fixed NullPointer in StreamingSessionActor](https://github.com/eclipse-ditto/ditto/pull/546) When closing a WebSocket session, a `NullPointerException` occurred which is fixed now. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_100M1a.md b/documentation/src/main/resources/pages/ditto/release_notes_100M1a.md index 11c17a627b9..c2e56e1757c 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_100M1a.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_100M1a.md @@ -11,52 +11,52 @@ This first milestone of the "1.0.0" release provides a preview of what to expect ## Changes -### [Add HiveMQ MQTT client as an alternative for MQTT integration](https://github.com/eclipse/ditto/pull/487) +### [Add HiveMQ MQTT client as an alternative for MQTT integration](https://github.com/eclipse-ditto/ditto/pull/487) -Due to the problems described in [#450](https://github.com/eclipse/ditto/issues/450) we decided to add the HiveMQ +Due to the problems described in [#450](https://github.com/eclipse-ditto/ditto/issues/450) we decided to add the HiveMQ MQTT Client as an alternative for the connection of Ditto to MQTT brokers. Once it has proven to be working and stable in production, it will replace the previous client (Alpakka/Paho). -### [Scalable event publishing](https://github.com/eclipse/ditto/pull/483) +### [Scalable event publishing](https://github.com/eclipse-ditto/ditto/pull/483) This patch improves horizontal scalability of Ditto's event publishing mechanism. Find a more detailed description of this change in the pull request. -### [Typed entity IDs](https://github.com/eclipse/ditto/pull/475) +### [Typed entity IDs](https://github.com/eclipse-ditto/ditto/pull/475) This change introduces validated Java representations for entity IDs. -### [Introduce architectural decision records (ADR)](https://github.com/eclipse/ditto/pull/470) +### [Introduce architectural decision records (ADR)](https://github.com/eclipse-ditto/ditto/pull/470) We want to keep track of architectural decisions and decided to use the format of ADRs for this purpose. -### [Relax uri restrictions](https://github.com/eclipse/ditto/pull/451) +### [Relax uri restrictions](https://github.com/eclipse-ditto/ditto/pull/451) With this change we allow more characters in uris. -### [Background cleanup for stale journal entries and snapshots](https://github.com/eclipse/ditto/pull/446) +### [Background cleanup for stale journal entries and snapshots](https://github.com/eclipse-ditto/ditto/pull/446) This change introduces a asynchronous background deletion approach which replaces the current approach. ## New features -### [Initial contribution of Java client SDK](https://github.com/eclipse/ditto-clients/pull/1) +### [Initial contribution of Java client SDK](https://github.com/eclipse-ditto/ditto-clients/pull/1) Contribution was extracted from former commercial-only client - all references to Bosch were removed. Consists of full working, OSGi capable "ditto-client" artifact. More information can be found in our [SDK documentation](client-sdk-java.html). -### [Configurable authorization servers](https://github.com/eclipse/ditto/pull/477) +### [Configurable authorization servers](https://github.com/eclipse-ditto/ditto/pull/477) Eclipse Ditto now supports all OAuth 2.0 providers which implement OpenID Connect out-of-the-box. See this [blog post](https://www.eclipse.org/ditto/2019-08-28-openid-connect.html) for more details. -### [Fine grained access for connections](https://github.com/eclipse/ditto/pull/463) +### [Fine grained access for connections](https://github.com/eclipse-ditto/ditto/pull/463) With this change it is possible to restrict access to connections on any level via a policy. -### [Reconnecting feature for created connections](https://github.com/eclipse/ditto/pull/442) +### [Reconnecting feature for created connections](https://github.com/eclipse-ditto/ditto/pull/442) When creating a connection, it will also contain the desired status of the connection. @@ -66,5 +66,5 @@ When creating a connection, it will also contain the desired status of the conne ### Various smaller bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.0.0-M1a). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.0.0-M1a). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_100M2.md b/documentation/src/main/resources/pages/ditto/release_notes_100M2.md index 36aaa6d2732..67764441e3c 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_100M2.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_100M2.md @@ -11,7 +11,7 @@ The second milestone is one last stop preparing for the upcoming "1.0.0" release ## Changes -### [Don't allow double slashes in JSON pointers and REST](https://github.com/eclipse/ditto/pull/524) +### [Don't allow double slashes in JSON pointers and REST](https://github.com/eclipse-ditto/ditto/pull/524) Ditto allowed to have double slashed in JSON pointers (e.g.: `features//foo/properties`) and HTTP endpoints. Allowing those did not always result the API behave like expected, so this is now handled more strictly. @@ -20,7 +20,7 @@ in a status code `400` (Bad request) at the HTTP API. ### Connection JSON format was adjusted -As a result of the newly added feature "[Map Hono device connection status to Thing feature](https://github.com/eclipse/ditto/issues/492)" +As a result of the newly added feature "[Map Hono device connection status to Thing feature](https://github.com/eclipse-ditto/ditto/issues/492)" (see below), the JSON format of connections was adjusted. The new format is documented here: @@ -32,7 +32,7 @@ The good news however is that previously created connections will automatically ## New features -### [Enhance Ditto's connectivity by invoking HTTP endpoints](https://github.com/eclipse/ditto/issues/491) +### [Enhance Ditto's connectivity by invoking HTTP endpoints](https://github.com/eclipse-ditto/ditto/issues/491) One of the bigger feature enhancements since the last milestone release is connections to existing HTTP endpoints/servers.
By adding the "connection type" HTTP to Ditto's connectivity feature Ditto can now perform HTTP calls for the configured @@ -41,7 +41,7 @@ By adding the "connection type" HTTP to Ditto's connectivity feature Ditto can n That may be used in order to integrate with other public HTTP APIs. See also the [published blog post about that](2019-10-17-http-connectivity.html). -### [Map Hono device connection status to Thing feature](https://github.com/eclipse/ditto/issues/492) +### [Map Hono device connection status to Thing feature](https://github.com/eclipse-ditto/ditto/issues/492) The integration with [Eclipse Hono](https://eclipse.org/hono/) was improved by adding the possibility tp extract `creation-time` and `ttd` headers from consumed Hono telemetry and event messages and automatically updating the @@ -51,14 +51,14 @@ This feature was added by enhancing and generalizing the overall [payload mappin as a result now multiple payload mappings may be defined and selectively applied to sources/targets in [connections](connectivity-manage-connections.html). -### [Support for OAuth based authentication in Ditto Java client](https://github.com/eclipse/ditto-clients/issues/17) +### [Support for OAuth based authentication in Ditto Java client](https://github.com/eclipse-ditto/ditto-clients/issues/17) In [1.0.0-M1a](release_notes_100-M1a.html) support for arbitrary OpenID Connect providers was [added](2019-08-28-openid-connect.html) to Ditto.
Now the Ditto Java client can authenticate itself by either providing "client-id" and "client-secret" or by supplying JWT tokens via a custom callback. -### [Throttle max. processed inbound websocket per time interval](https://github.com/eclipse/ditto/pull/517) +### [Throttle max. processed inbound websocket per time interval](https://github.com/eclipse-ditto/ditto/pull/517) It is not always desirable that a single websocket connection may "flood" a Ditto backend with a massive amount of commands.
@@ -68,12 +68,12 @@ The defaults configuration is: 100 / 1second ## Bugfixes -### [Deleted things were still available in search](https://github.com/eclipse/ditto/issues/526) +### [Deleted things were still available in search](https://github.com/eclipse-ditto/ditto/issues/526) When things were deleted, they were still available and findable in the search index (as the search index was cleared via a `TTL` index). This is now fixed. -### [Failed connections AMQP 1.0 target didn't back-off](https://github.com/eclipse/ditto/pull/516) +### [Failed connections AMQP 1.0 target didn't back-off](https://github.com/eclipse-ditto/ditto/pull/516) When an AMQP 1.0 endpoint did not allow to open a link to an address, Ditto tried to reconnect in a high frequency. As such errors are most likely configuration errors, that almost never is a good solution. The aggressive reconnect @@ -83,5 +83,5 @@ This fix introduces an exponential back-off mechanism. ### Various smaller bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.0.0-M2). +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.0.0-M2). diff --git a/documentation/src/main/resources/pages/ditto/release_notes_110.md b/documentation/src/main/resources/pages/ditto/release_notes_110.md index e452ba8cb2d..48462d47939 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_110.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_110.md @@ -8,7 +8,7 @@ permalink: release_notes_110.html The first minor (feature adding) release of Eclipse Ditto 1 is finally here: **1.1.0**. -It is API and [binary compatible](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +It is API and [binary compatible](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) to Eclipse Ditto 1.0.0. ## Changelog @@ -19,16 +19,16 @@ bugfixes were added. ### Changes -#### [Java 11 as runtime environment](https://github.com/eclipse/ditto/issues/308) +#### [Java 11 as runtime environment](https://github.com/eclipse-ditto/ditto/issues/308) The default Java runtime for Ditto's Docker containers was switched from Java 8 to Java 11 which should have some benefits in storing Strings in memory (this was already added in Java 9). Language features of newer Java versions can now be used in the "services" part of Ditto, the Java APIs and models relevant -for [semantic versioning](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +for [semantic versioning](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) are still compatible to Java 8. -#### [CBOR as Ditto internal serialization provider](https://github.com/eclipse/ditto/pull/598) +#### [CBOR as Ditto internal serialization provider](https://github.com/eclipse-ditto/ditto/pull/598) As a bachelor thesis, [Erik Escher](https://github.com/erikescher) evaluated mechanisms to improve the serialization overhead done in Ditto clusters. @@ -37,7 +37,7 @@ His findings using [CBOR](https://cbor.io) as an alternative to plain JSON resul roundtrip times and throughput. The Ditto team was happy to accept his pull request, again improving overall performance in Ditto. -#### [More strict Content-Type parsing for HTTP request payload](https://github.com/eclipse/ditto/pull/650) +#### [More strict Content-Type parsing for HTTP request payload](https://github.com/eclipse-ditto/ditto/pull/650) In the past, Ditto did not evaluate the HTTP `Content-Type` header of HTTP requests sending along payload. As this can be a potential security issue (e.g. in scope of CORS requests), the `Content-Type` is now strictly enforced to @@ -46,16 +46,16 @@ be of `application/json` wherever Ditto only accepts JSON request payload. ### New features -#### [Management of policies via Ditto Protocol and in Java client](https://github.com/eclipse/ditto/issues/554) +#### [Management of policies via Ditto Protocol and in Java client](https://github.com/eclipse-ditto/ditto/issues/554) The [policy](basic-policy.html) entities can now - in addition to their HTTP API - be managed via the [Ditto Protocol](protocol-specification-policies.html). That means also via [WebSocket](httpapi-protocol-bindings-websocket.html) and [connections](basic-connections.html) (e.g. AMQP, MQTT, ..). APIs for [policy management](client-sdk-java.html#manage-policies) were also added to the -[Ditto Java Client](https://github.com/eclipse/ditto-clients/pull/46). +[Ditto Java Client](https://github.com/eclipse-ditto/ditto-clients/pull/46). -#### [Searching things via Ditto Protocol and in Java client](https://github.com/eclipse/ditto/issues/575) +#### [Searching things via Ditto Protocol and in Java client](https://github.com/eclipse-ditto/ditto/issues/575) New [Ditto Protocol for search](protocol-specification-things-search.html) was added in order to define a search query via the Ditto Protocol and also get results via an asynchronous channel. As a result, searching for things is now also @@ -63,9 +63,9 @@ possible via [WebSocket](httpapi-protocol-bindings-websocket.html) and [connecti (e.g. AMQP, MQTT, ..). APIs for [searching things](client-sdk-java.html#search-for-things) were also added to the -[Ditto Java Client](https://github.com/eclipse/ditto-clients/pull/53). +[Ditto Java Client](https://github.com/eclipse-ditto/ditto-clients/pull/53). -#### [Enriching messages and events before publishing to external subscribers](https://github.com/eclipse/ditto/issues/561) +#### [Enriching messages and events before publishing to external subscribers](https://github.com/eclipse-ditto/ditto/issues/561) When subscribing [change notifications](basic-changenotifications.html) or for messages to publish to external system or deliver via [WebSocket](httpapi-protocol-bindings-websocket.html) it is now possible to [enrich](basic-enrichment.html) @@ -75,9 +75,9 @@ This can be useful when e.g. only a sensor value of a device changes, but your a additional context of the affected thing (e.g. a location which does not change with each sensor update). APIs for [enriching changes](client-sdk-java.html#subscribe-to-enriched-change-notifications) were also added to the -[Ditto Java Client](https://github.com/eclipse/ditto-clients/pull/43). +[Ditto Java Client](https://github.com/eclipse-ditto/ditto-clients/pull/43). -#### [Establish connections to MQTT 5 brokers](https://github.com/eclipse/ditto/issues/561) +#### [Establish connections to MQTT 5 brokers](https://github.com/eclipse-ditto/ditto/issues/561) The Ditto community (namely [Alexander Wellbrock (w4tsn)](https://github.com/w4tsn) from [othermo GmbH](https://www.othermo.de)) contributed MQTT 5 support to Ditto's connectivity capabilities.
@@ -86,19 +86,19 @@ With that is is possible to also establish connections to MQTT 5 brokers and eve Thank you very much for this great contribution. -#### [End-2-end acknowledgements](https://github.com/eclipse/ditto/issues/611) +#### [End-2-end acknowledgements](https://github.com/eclipse-ditto/ditto/issues/611) Until now, messages consumed by Eclipse Ditto were processed without a guarantee. That is being addressed with this first feature addition, the model and logic in order to request and emit [acknowledgements](basic-acknowledgements.html). -The follow-up issue [#661](https://github.com/eclipse/ditto/issues/661) will automatically handle acknowledgements +The follow-up issue [#661](https://github.com/eclipse-ditto/ditto/issues/661) will automatically handle acknowledgements in Ditto managed connections, configured for connection sources and targets, providing QoS 1 (at least once) semantic for message processing in Ditto via connections. APIs for [requesting and issuing acknowledgements](client-sdk-java.html#request-and-issue-acknowledgements) were also -added to the [Ditto Java Client](https://github.com/eclipse/ditto-clients/pull/56). +added to the [Ditto Java Client](https://github.com/eclipse-ditto/ditto-clients/pull/56). -#### [Pre-authenticated authentication mechanism](https://github.com/eclipse/ditto/issues/560) +#### [Pre-authenticated authentication mechanism](https://github.com/eclipse-ditto/ditto/issues/560) Officially added+[documented](installation-operating.html#pre-authentication) support of how Ditto external authentication providers may be configured to authenticate users in Ditto by adding them as an HTTP reverse proxy in @@ -107,7 +107,7 @@ front of Ditto. ### Deprecations -#### [API version 1 deprecation](https://github.com/eclipse/ditto/pull/608) +#### [API version 1 deprecation](https://github.com/eclipse-ditto/ditto/pull/608) Now that Ditto has a full replacement for ACLs, namely [policies](basic-policy.html) which now can also be managed via the [Ditto Protocol](protocol-specification-policies.html) and the @@ -125,7 +125,7 @@ So when you start using Ditto, please make sure to use API version `2` (using po Several bugs in Ditto 1.0.0 were fixed for 1.1.0.
This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.1.0), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.1.0), including the fixed bugs. ## Migration notes diff --git a/documentation/src/main/resources/pages/ditto/release_notes_111.md b/documentation/src/main/resources/pages/ditto/release_notes_111.md index bb38587e4fb..26596dd4bff 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_111.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_111.md @@ -15,13 +15,13 @@ Compared to the latest release [1.1.0](release_notes_110.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.1.1), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.1.1), including the fixed bugs. -#### [Allow to respond to live messages via ditto client](https://github.com/eclipse/ditto-clients/pull/60) +#### [Allow to respond to live messages via ditto client](https://github.com/eclipse-ditto/ditto-clients/pull/60) Don't force response-required to be false when calling send() without response consumer. -#### [Header mapping](https://github.com/eclipse/ditto/pull/671) +#### [Header mapping](https://github.com/eclipse-ditto/ditto/pull/671) Remove filtering of unknown headers for adaptables. We need those headers for header mapping in connectivity. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_112.md b/documentation/src/main/resources/pages/ditto/release_notes_112.md index 91efd48c483..fad6efd18ba 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_112.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_112.md @@ -14,7 +14,7 @@ Compared to the latest release [1.1.1](release_notes_111.html), the following ch ### Changes -#### [Publish minor and micro version tags to Docker Hub](https://github.com/eclipse/ditto/pull/693) +#### [Publish minor and micro version tags to Docker Hub](https://github.com/eclipse-ditto/ditto/pull/693) Starting with Ditto 1.1.2, the Docker images built and pushed to Docker Hub are: * full version (e.g. `1.1.2`) @@ -25,17 +25,17 @@ Starting with Ditto 1.1.2, the Docker images built and pushed to Docker Hub are: ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.1.2), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.1.2), including the fixed bugs. -#### [Fix NullPointerException when disabling validation of certificates in connections](https://github.com/eclipse/ditto/pull/688) +#### [Fix NullPointerException when disabling validation of certificates in connections](https://github.com/eclipse-ditto/ditto/pull/688) -Fixed [Mqtt 3 connection without using certificate validation](https://github.com/eclipse/ditto/issues/679). +Fixed [Mqtt 3 connection without using certificate validation](https://github.com/eclipse-ditto/ditto/issues/679). -#### [Connection creation timeout](https://github.com/eclipse/ditto/pull/692) +#### [Connection creation timeout](https://github.com/eclipse-ditto/ditto/pull/692) There was an issue where sometimes the creation of a connection entity failed with a timeout just because the establishing of the actual connection took too much time. -#### [Minor improvements to throughput/performance](https://github.com/eclipse/ditto/pull/689) +#### [Minor improvements to throughput/performance](https://github.com/eclipse-ditto/ditto/pull/689) Some minor overall simplifications and performance improvements. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_113.md b/documentation/src/main/resources/pages/ditto/release_notes_113.md index 5d75f90a6ed..857d9eb5e4a 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_113.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_113.md @@ -15,29 +15,29 @@ Compared to the latest release [1.1.2](release_notes_112.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.1.3), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.1.3), including the fixed bugs. -#### [Do not decided based on the response if it was required](https://github.com/eclipse/ditto/pull/734) +#### [Do not decided based on the response if it was required](https://github.com/eclipse-ditto/ditto/pull/734) -Fixed [Responses should not decide whether they're required](https://github.com/eclipse/ditto/issues/677). +Fixed [Responses should not decide whether they're required](https://github.com/eclipse-ditto/ditto/issues/677). -#### [Include fields query parameter when retrieving feature properties](https://github.com/eclipse/ditto/pull/727) +#### [Include fields query parameter when retrieving feature properties](https://github.com/eclipse-ditto/ditto/pull/727) There was an issue where the fields query parameter wasn't taken into account when retrieving feature properties. -#### [Delegate default options from wrapping message mapper to wrapped message mapper](https://github.com/eclipse/ditto/pull/723) +#### [Delegate default options from wrapping message mapper to wrapped message mapper](https://github.com/eclipse-ditto/ditto/pull/723) The WrappingMessageMapper didn't delegate the default options to the message mapper it was wrapping. -#### [Improved error message for unknown/invalid host names in a connection configuration](https://github.com/eclipse/ditto/pull/676) +#### [Improved error message for unknown/invalid host names in a connection configuration](https://github.com/eclipse-ditto/ditto/pull/676) An unknown/invalid host in a connection configuration caused an exception with an error message that did not indicate the actual cause. -#### [Reworked reconnect behaviour of java client](https://github.com/eclipse/ditto-clients/pull/64) +#### [Reworked reconnect behaviour of java client](https://github.com/eclipse-ditto/ditto-clients/pull/64) There were reported issues with the reconnecting behaviour of the java client. We improved the reconnecting behaviour, so it should be more reliable. -#### [Added org.reactivestreams to osgi imports](https://github.com/eclipse/ditto-clients/pull/73) +#### [Added org.reactivestreams to osgi imports](https://github.com/eclipse-ditto/ditto-clients/pull/73) The package `org.reactivestreams` was missing in the OSGI imports of our java client. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_115.md b/documentation/src/main/resources/pages/ditto/release_notes_115.md index f837808dd64..9e9ba3d183f 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_115.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_115.md @@ -18,10 +18,10 @@ Compared to the latest release [1.1.3](release_notes_113.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.1.5), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.1.5), including the fixed bugs. -#### [Compile model modules with Java 8](https://github.com/eclipse/ditto/pull/769) +#### [Compile model modules with Java 8](https://github.com/eclipse-ditto/ditto/pull/769) The 2 modules in the Ditto model were accidentally compiled with Java 11 as source/target which caused that e.g. the Ditto client could not be used any longer with Java 8: diff --git a/documentation/src/main/resources/pages/ditto/release_notes_120.md b/documentation/src/main/resources/pages/ditto/release_notes_120.md index a5364cb6d92..fa6d73f9bb9 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_120.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_120.md @@ -9,7 +9,7 @@ permalink: release_notes_120.html The second minor (feature adding) release of Eclipse Ditto 1 is here: **1.2.0**. -It is API and [binary compatible](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +It is API and [binary compatible](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) to Eclipse Ditto 1.0.0 and 1.1.0. ## Changelog @@ -21,7 +21,7 @@ The main change of Ditto 1.2.0 is the now full support for QoS 1 ("at least once ### Changes -#### [Update Docker base image to OpenJ9 0.21.0](https://github.com/eclipse/ditto/pull/743) +#### [Update Docker base image to OpenJ9 0.21.0](https://github.com/eclipse-ditto/ditto/pull/743) Updated to running the Ditto Docker containers with the latest OpenJ9 0.21.0 (with OpenJDK 11). @@ -37,45 +37,45 @@ environment variable `CONNECTIVITY_MESSAGE_MAPPING_MAX_POOL_SIZE`). ### New features -#### [`fn:filter` function for connectivity header mapping](https://github.com/eclipse/ditto/pull/674) +#### [`fn:filter` function for connectivity header mapping](https://github.com/eclipse-ditto/ditto/pull/674) In connection [header mappings](connectivity-header-mapping.html) as part of the placeholders, the new [`fn:filter`](basic-placeholders.html#function-library) function may be used in order to remove the result of the previous expression in the function pipeline unless the condition specified by the parameters is satisfied. -#### [Whoami HTTP resource](https://github.com/eclipse/ditto/pull/687) +#### [Whoami HTTP resource](https://github.com/eclipse-ditto/ditto/pull/687) The new HTTP `GET` resource `/whoami` may be called in order to find out which authorization subjects were resolved in the HTTP call's authentication. This can be e.g. useful to find out the used JWT subject which should be added to [policies](basic-policy.html#subjects). -#### [Support using client certificate based authentication in HTTP push connections](https://github.com/eclipse/ditto/pull/695) +#### [Support using client certificate based authentication in HTTP push connections](https://github.com/eclipse-ditto/ditto/pull/695) Connections of type [HTTP push](connectivity-protocol-bindings-http.html) can now, additionally to username/password based authentication, make use of [client certificate based authentication](connectivity-protocol-bindings-http.html#client-certificate-authentication). -#### [Automatic end-2-end acknowledgements handling for managed connections](https://github.com/eclipse/ditto/issues/661) +#### [Automatic end-2-end acknowledgements handling for managed connections](https://github.com/eclipse-ditto/ditto/issues/661) Acknowledgements can now be configured to be requested for messages consumed by connection [sources (acknowledgement requests)](basic-connections.html#source-acknowledgement-requests) and can automatically be issued by targets to automatically [issue acknowledgements](basic-connections.html#target-issued-acknowledgement-label) for all published twin events, live commands and live messages that request them. -#### [End-2-end acknowledgements support for "live" messages/commands](https://github.com/eclipse/ditto/issues/757) +#### [End-2-end acknowledgements support for "live" messages/commands](https://github.com/eclipse-ditto/ditto/issues/757) Acknowledgements for [live messages/commands](basic-acknowledgements.html#assure-qos-until-processing-of-a-live-commandmessage-by-a-subscriber---live-response) are now supported as well. Both requesting and issuing them, e.g. in order to acknowledge that a message was successfully received without directly responding to it. -#### [Addition of `_created` date to things](https://github.com/eclipse/ditto/issues/749) +#### [Addition of `_created` date to things](https://github.com/eclipse-ditto/ditto/issues/749) Whenever a [thing](basic-thing.html) is now created, a JSON field `"_created"` is now added containing the creation date. This field can be selected via [fields selection](httpapi-concepts.html#with-field-selector), as the already existing `"_modified"` field can also be. The created date can also be used as part of [search RQL queries](basic-rql.html). -#### [Support for adding `_metadata` to things](https://github.com/eclipse/ditto/issues/680) +#### [Support for adding `_metadata` to things](https://github.com/eclipse-ditto/ditto/issues/680) On modifying API calls to a [thing](basic-thing.html), additional metadata can now be passed with the header field `"put-header"`. Documentation for this feature is still missing, but will be added soon after the 1.2.0 release. @@ -84,9 +84,9 @@ On modifying API calls to a [thing](basic-thing.html), additional metadata can n Several bugs in Ditto 1.1.x were fixed for 1.2.0.
This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.2.0), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.2.0), including the fixed bugs. -#### [Connectivity service does not consume message after reconnect to AMQP (0.9.1)](https://github.com/eclipse/ditto/issues/770) +#### [Connectivity service does not consume message after reconnect to AMQP (0.9.1)](https://github.com/eclipse-ditto/ditto/issues/770) Connections via AMQP 0.9.1 did not correctly resume message consumption after the broker was e.g. restarted. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_121.md b/documentation/src/main/resources/pages/ditto/release_notes_121.md index 050e00eee1a..81dd65ad3a5 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_121.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_121.md @@ -16,10 +16,10 @@ Compared to the latest release [1.2.0](release_notes_120.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.2.1), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.2.1), including the fixed bugs. -#### [Ditto JSON is not OSGi compatible due to missing imports](https://github.com/eclipse/ditto/issues/790) +#### [Ditto JSON is not OSGi compatible due to missing imports](https://github.com/eclipse-ditto/ditto/issues/790) The OSGi bundle `ditto-json` was not compatible to be run in OSGi environments as imports of 3rd party libraries were used which were not defined in the `Import-Package` of the bundle. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_130.md b/documentation/src/main/resources/pages/ditto/release_notes_130.md index fe6c5c04949..ee52e23efad 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_130.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_130.md @@ -7,7 +7,7 @@ summary: "Version 1.3.0 of Eclipse Ditto, released on 30.09.2020" permalink: release_notes_130.html --- -Ditto **1.3.0** is API and [binary compatible](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +Ditto **1.3.0** is API and [binary compatible](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) to prior Eclipse Ditto 1.x versions. ## Changelog @@ -18,18 +18,18 @@ bugfixes were added. ### Changes -#### [Update Akka, Akka HTTP and Scala to latest versions](https://github.com/eclipse/ditto/issues/774) +#### [Update Akka, Akka HTTP and Scala to latest versions](https://github.com/eclipse-ditto/ditto/issues/774) The core libraries Ditto is built on were updated to their latest versions which should improve cluster stability and overall performance. -#### [Removed OWASP security headers](https://github.com/eclipse/ditto/pull/804) +#### [Removed OWASP security headers](https://github.com/eclipse-ditto/ditto/pull/804) Setting the [OWASP recommended secure HTTP headers](https://owasp.org/www-project-secure-headers/) (e.g. `X-Frame-Options`, `X-Content-Type-Options`, `X-XSS-Protection`) was removed from the Ditto codebase as such headers are typically set in a reverse proxy (e.g. nginx) or in a cloud loadbalancer in front of Ditto. -#### [Ditto Java Client: Changed default initial reconnect behavior](https://github.com/eclipse/ditto-clients/pull/86) +#### [Ditto Java Client: Changed default initial reconnect behavior](https://github.com/eclipse-ditto/ditto-clients/pull/86) A newly created configuration was added whether a Ditto Java Client should retry connecting to a Ditto backend even when the initial connection attempt failed @@ -49,7 +49,7 @@ connection should fail with an exception instead, so this is the new default beh ### New features -#### [Automatic creation of things](https://github.com/eclipse/ditto/issues/760) +#### [Automatic creation of things](https://github.com/eclipse-ditto/ditto/issues/760) Added a [payload mapper](connectivity-mapping.html) for connectivity which implicitly creates a new digital twin (thing) for incoming messages: [ImplicitThingCreation Mapper](connectivity-mapping.html#implicitthingcreation-mapper). @@ -60,7 +60,7 @@ automatically creates connected devices, for example when a new device connects This new feature can work together with the [Hono feature for implicit registration of devices connected via gateways](https://github.com/eclipse/hono/issues/2053). -#### [Use response of HTTP push connections as live message response](https://github.com/eclipse/ditto/pull/809) +#### [Use response of HTTP push connections as live message response](https://github.com/eclipse-ditto/ditto/pull/809) When [HTTP connections](connectivity-protocol-bindings-http.html) there are now several options to respond to published [live messages](protocol-specification-things-messages.html): @@ -69,7 +69,7 @@ to respond to published [live messages](protocol-specification-things-messages.h For example, it is possible to use the HTTP response of the foreign HTTP endpoint (Webhook) as Ditto live message response. -#### [Raw payload mapper to enable raw pass-through of live messages](https://github.com/eclipse/ditto/issues/777) +#### [Raw payload mapper to enable raw pass-through of live messages](https://github.com/eclipse-ditto/ditto/issues/777) Added a [payload mapper](connectivity-mapping.html) for connectivity which converts consumed messages via connectivity in "raw mode": [RawMessage mapper](connectivity-mapping.html#rawmessage-mapper). @@ -85,21 +85,21 @@ WebSocket. Several bugs in Ditto 1.2.x were fixed for 1.3.0.
This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.3.0), including the fixed bugs.
-Here as well for the Ditto Java Client: [merged pull requests](https://github.com/eclipse/ditto-clients/pulls?q=is%3Apr+milestone%3A1.3.0) +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.3.0), including the fixed bugs.
+Here as well for the Ditto Java Client: [merged pull requests](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is%3Apr+milestone%3A1.3.0) -#### [Responses from HTTP /messages API were JSON escaped](https://github.com/eclipse/ditto/issues/805) +#### [Responses from HTTP /messages API were JSON escaped](https://github.com/eclipse-ditto/ditto/issues/805) With Ditto 1.2.0 HTTP responses to the `POST /messages` APIs which transported `application/json` were falsely JSON escaped. As the fix for that had to be done in several steps and at several places, the fix is not backported to the Ditto 1.2.0 line and it is suggested to update to Ditto 1.3.0 right away, if affected by this bug. -#### [Putting `_metadata` while creating a Thing does not work bug](https://github.com/eclipse/ditto/issues/801) +#### [Putting `_metadata` while creating a Thing does not work bug](https://github.com/eclipse-ditto/ditto/issues/801) When putting [Metadata](basic-metadata.html) as part of a "create thing" API call, the metadata was not applied. Only when updating an existing thing, the metadata was applied. -#### [Ditto Java Client: threads leakage](https://github.com/eclipse/ditto-clients/pull/87) +#### [Ditto Java Client: threads leakage](https://github.com/eclipse-ditto/ditto-clients/pull/87) The Ditto Java client did not close/cleanup its threadpools when closing the client. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_140.md b/documentation/src/main/resources/pages/ditto/release_notes_140.md index dedc5f167f2..489874d8c44 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_140.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_140.md @@ -7,7 +7,7 @@ summary: "Version 1.4.0 of Eclipse Ditto, released on 28.10.2020" permalink: release_notes_140.html --- -Ditto **1.4.0** is API and [binary compatible](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +Ditto **1.4.0** is API and [binary compatible](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) to prior Eclipse Ditto 1.x versions. ## Changelog @@ -18,22 +18,22 @@ bugfixes were added. ### Changes -#### [Status codes of live responses are no longer interpreted for acknowledgement requests](https://github.com/eclipse/ditto/pull/833) +#### [Status codes of live responses are no longer interpreted for acknowledgement requests](https://github.com/eclipse-ditto/ditto/pull/833) Senders of live responses may freely choose the status code, since it no longer affects the technical settlement and redelivery of the corresponding live commands. -#### [The header response-required is always set to false for responses and events](https://github.com/eclipse/ditto/pull/850) +#### [The header response-required is always set to false for responses and events](https://github.com/eclipse-ditto/ditto/pull/850) Ditto sets the header `response-required` to `false` for signals that do not anticipate any responses, so that the header has a consistent meaning regardless of signal type. -#### [OCSP is optional for connections](https://github.com/eclipse/ditto/pull/854) +#### [OCSP is optional for connections](https://github.com/eclipse-ditto/ditto/pull/854) Ditto will establish a connection even if the broker has a revoked certificate according to OCSP. Failed revocation checks will generate warnings in the connection log. This is to guard against unavailability of OCSP servers. -#### [Placeholder topic:entityId renamed to topic:entityName](https://github.com/eclipse/ditto/pull/859) +#### [Placeholder topic:entityId renamed to topic:entityName](https://github.com/eclipse-ditto/ditto/pull/859) The placeholder `topic:entityId` was not named correctly. It was resolved with the name of an entity and not the complete ID. Therefore, a new placeholder @@ -41,36 +41,36 @@ name of an entity and not the complete ID. Therefore, a new placeholder ### New features -#### [Acknowledgement label declaration](https://github.com/eclipse/ditto/issues/792) +#### [Acknowledgement label declaration](https://github.com/eclipse-ditto/ditto/issues/792) Each subscriber of Ditto signals by Websocket or other connections is required to declare the labels of acknowledgements it may send. The labels should be unique to the subscriber. Labels of acknowledgements sent via a connection source or issued by a connection target must be prefixed by the connection ID followed by a colon. This is to prevent racing in fulfillment of acknowledgement requests and to detect misconfiguration early. -Acknowledgement label declaration is available in [Ditto java client](https://github.com/eclipse/ditto-clients/pull/98). +Acknowledgement label declaration is available in [Ditto java client](https://github.com/eclipse-ditto/ditto-clients/pull/98). ### Bugfixes Several bugs in Ditto 1.3.0 were fixed for 1.4.0.
This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.4.0), including the fixed bugs.
-Here as well for the Ditto Java Client: [merged pull requests](https://github.com/eclipse/ditto-clients/pulls?q=is%3Apr+milestone%3A1.4.0) +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.4.0), including the fixed bugs.
+Here as well for the Ditto Java Client: [merged pull requests](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is%3Apr+milestone%3A1.4.0) -#### [Thread-safe loggers added](https://github.com/eclipse/ditto/issues/773) +#### [Thread-safe loggers added](https://github.com/eclipse-ditto/ditto/issues/773) Concurrency issues in Ditto loggers and logging adapters are addressed by introducing thread-safe variants. -#### [Search via SSE enabled](https://github.com/eclipse/ditto/issues/822) +#### [Search via SSE enabled](https://github.com/eclipse-ditto/ditto/issues/822) Search via SSE was disabled due to incorrect initialization. It is enabled again. -#### [Memory consumption of outgoing AMQP connections limited](https://github.com/eclipse/ditto/pull/853) +#### [Memory consumption of outgoing AMQP connections limited](https://github.com/eclipse-ditto/ditto/pull/853) AMQP 1.0 connections to a slow broker could accumulate indefinitely messages yet to be published. Now only a fixed number of messages are retained. -#### [Java client: Message ordering fixed](https://github.com/eclipse/ditto-clients/pull/97) +#### [Java client: Message ordering fixed](https://github.com/eclipse-ditto/ditto-clients/pull/97) There was a bug in Ditto Java client that may cause messages to be handled in a different order than when they are received. It made some search results look empty when they are not. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_150.md b/documentation/src/main/resources/pages/ditto/release_notes_150.md index 7a8e2073e09..b94a01e3adb 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_150.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_150.md @@ -7,7 +7,7 @@ summary: "Version 1.5.0 of Eclipse Ditto, released on 10.12.2020" permalink: release_notes_150.html --- -Ditto **1.5.0** is API and [binary compatible](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +Ditto **1.5.0** is API and [binary compatible](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) to prior Eclipse Ditto 1.x versions. ## Changelog @@ -21,7 +21,7 @@ bugfixes were added. ### Changes -#### [Negatively settling processed AMQP 1.0 messages changed to `rejected`](https://github.com/eclipse/ditto/pull/907) +#### [Negatively settling processed AMQP 1.0 messages changed to `rejected`](https://github.com/eclipse-ditto/ditto/pull/907) In previous versions, Ditto negatively settled messages consumed via AMQP 1.0 which could not be applied to Ditto (e.g. because the received message could not be understood or permissions were missing) with `modified[undeliverable-here]`. @@ -30,22 +30,22 @@ This was changed to settle with `rejected` (see [AMQP 1.0 spec](http://docs.oasis-open.org/amqp/core/v1.0/os/amqp-core-messaging-v1.0-os.html#type-rejected)) instead, as this is the more correct settlement outcome. -#### [MongoDB for unit tests was increased to version 4.2](https://github.com/eclipse/ditto/pull/896) +#### [MongoDB for unit tests was increased to version 4.2](https://github.com/eclipse-ditto/ditto/pull/896) Prior to Ditto 1.5.0, the unit tests still were done against MongoDB version 3.6 which reaches end-of-life in April 2021. -#### [Config files consolidation](https://github.com/eclipse/ditto/pull/888) +#### [Config files consolidation](https://github.com/eclipse-ditto/ditto/pull/888) No special `-cloud.conf` and `-docker.conf` config files are needed any longer, there are only special config files ending with `-dev.conf` which contain configuration in order to start Eclipse Ditto e.g. locally in an IDE. ### New features -#### [Header mapping for Feature ID in connectivity](https://github.com/eclipse/ditto/issues/857) +#### [Header mapping for Feature ID in connectivity](https://github.com/eclipse-ditto/ditto/issues/857) Feature IDs may now be used as placeholders in [Connectivity header mappings](basic-placeholders.html#scope-connections). -#### [Addition of "desired" feature properties in model and APIs](https://github.com/eclipse/ditto/issues/697) +#### [Addition of "desired" feature properties in model and APIs](https://github.com/eclipse-ditto/ditto/issues/697) A feature which was long on the roadmap of Eclipse Ditto is the ability to distinguish between reported and [desired twin state](basic-feature.html#feature-desired-properties). @@ -56,7 +56,7 @@ for a property. This issue layed the foundation by creating the model and the APIs in order to manage those `desiredProperties`. -#### [Issuing "weak acknowledgements" when a command requesting acks was filtered out](https://github.com/eclipse/ditto/issues/852) +#### [Issuing "weak acknowledgements" when a command requesting acks was filtered out](https://github.com/eclipse-ditto/ditto/issues/852) When using [acknowledgements](basic-acknowledgements.html) in order to guarantee "at least once" (QoS 1) delivery and scenarios like: @@ -66,7 +66,7 @@ scenarios like: resending the signal will not help. Ditto now emits a "weak acknowledgement" for such cases that does not trigger redelivery. -#### [Ditto internal pub/sub supports using a "grouping" concept](https://github.com/eclipse/ditto/issues/878) +#### [Ditto internal pub/sub supports using a "grouping" concept](https://github.com/eclipse-ditto/ditto/issues/878) A "group" concept was added to Ditto pub/sub: * Subscribers may subscribe with a group name. @@ -75,7 +75,7 @@ A "group" concept was added to Ditto pub/sub: With this feature, the event publishing at connections will scale with the number of client actors by having the client actors subscribe for events directly using the connection ID as group. -#### [Addition of "cloudevents" HTTP endpoint](https://github.com/eclipse/ditto/issues/889) +#### [Addition of "cloudevents" HTTP endpoint](https://github.com/eclipse-ditto/ditto/issues/889) While [cloud events](https://cloudevents.io) provide bindings for Kafka, MQTT, ... they also have an HTTP endpoint binding, which can easily be used in the combination with Knative. @@ -91,10 +91,10 @@ for this addition to Eclipse Ditto. Several bugs in Ditto 1.4.0 were fixed for 1.5.0.
This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.5.0), including the fixed bugs.
-Here as well for the Ditto Java Client: [merged pull requests](https://github.com/eclipse/ditto-clients/pulls?q=is%3Apr+milestone%3A1.5.0) +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.5.0), including the fixed bugs.
+Here as well for the Ditto Java Client: [merged pull requests](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is%3Apr+milestone%3A1.5.0) -#### [Fix that sending messages with non-existing "value" was not possible via HTTP endpoints](https://github.com/eclipse/ditto/pull/875) +#### [Fix that sending messages with non-existing "value" was not possible via HTTP endpoints](https://github.com/eclipse-ditto/ditto/pull/875) The HTTP `/messages` endpoints did not allow that a Ditto Protocol messages with non-existing `"value"` were created for HTTP invocations which did not include payload at all. @@ -102,14 +102,14 @@ HTTP invocations which did not include payload at all. That was fixed in the way that for requests with `Content-Length: 0` the `"value"` is now removed from the resulting Ditto Protocol message instead of being `"value": ""` (empty JSON string). -#### [Ditto Java client: When starting consumption with invalid filter, wrongly timeout exception is propagate to the user](https://github.com/eclipse/ditto-clients/pull/105) +#### [Ditto Java client: When starting consumption with invalid filter, wrongly timeout exception is propagate to the user](https://github.com/eclipse-ditto/ditto-clients/pull/105) `dittoClient.twin().startConsumption(org.eclipse.ditto.client.options.Options.Consumption.filter("invalidFilter"))` throwed a wrong exception and did not propagate the real error to the user. -Affected [Ditto PR](https://github.com/eclipse/ditto/pull/902). +Affected [Ditto PR](https://github.com/eclipse-ditto/ditto/pull/902). -#### [Ditto Java client: Fix FeatureChange consumption for specific feature change-registration](https://github.com/eclipse/ditto-clients/pull/101) +#### [Ditto Java client: Fix FeatureChange consumption for specific feature change-registration](https://github.com/eclipse-ditto/ditto-clients/pull/101) This fixes a bug that caused ignoring features in a FeatureChange for change-registrations on single features, when only a single subpath exists in the feature (i.e. feature with only properties). @@ -119,7 +119,7 @@ when only a single subpath exists in the feature (i.e. feature with only propert ### MongoDB hostname configuration -Due to the [consolidation of config files](https://github.com/eclipse/ditto/pull/888), it is now **required to configure +Due to the [consolidation of config files](https://github.com/eclipse-ditto/ditto/pull/888), it is now **required to configure the MongoDB `hostname` explicitly** as the default hostname was changed to `localhost`.
Previously, this hostname was automatically set to `mongodb` (which is the hostname of the MongoDB when e.g. the `docker-compose.yaml` deployment is used) in Docker based environments. @@ -127,4 +127,4 @@ Previously, this hostname was automatically set to `mongodb` (which is the hostn This now has to be manually done via the environment variable `MONGO_DB_HOSTNAME`. The default `docker-compose.yaml` was also adjusted accordingly: -[docker-compose.yml](https://github.com/eclipse/ditto/blob/master/deployment/docker/docker-compose.yml) +[docker-compose.yml](https://github.com/eclipse-ditto/ditto/blob/master/deployment/docker/docker-compose.yml) diff --git a/documentation/src/main/resources/pages/ditto/release_notes_151.md b/documentation/src/main/resources/pages/ditto/release_notes_151.md index dff517cf0b2..ffed300e2fd 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_151.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_151.md @@ -16,4 +16,4 @@ Compared to the latest release [1.5.0](release_notes_150.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A1.5.1), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A1.5.1), including the fixed bugs. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_200.md b/documentation/src/main/resources/pages/ditto/release_notes_200.md index 1ae02d1b642..43ea9d5f524 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_200.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_200.md @@ -72,7 +72,7 @@ pointing to an alternative implementation to use instead. Now, these deprecated Some changes to the codebase which could not be done in Ditto 1.x without breaking binary compatibility were also done. -#### [Removed content-type header mapping for connection targets](https://github.com/eclipse/ditto/pull/934) +#### [Removed content-type header mapping for connection targets](https://github.com/eclipse-ditto/ditto/pull/934) Removed the default header mapping of `content-type` for new connection targets. The header mapping led to irritating results, when payload mapping and header mapping disagreed on the actual `content-type`. Existing connections will still @@ -83,7 +83,7 @@ If you need to keep the old behavior, please have a look at the #### OpenID Connect configuration change -For supporting [Configurable OpenID Connect / OAuth2.0 claim extraction](https://github.com/eclipse/ditto/issues/512), +For supporting [Configurable OpenID Connect / OAuth2.0 claim extraction](https://github.com/eclipse-ditto/ditto/issues/512), the configuration format was changed, please have a look at the [migration notes](#openid-connect-configuration-for-gateway). @@ -205,7 +205,7 @@ In addition, all APIs which returned a `CompletableFuture` were adjusted to retu #### Ditto JavaScript client -Starting with Ditto 2.0.0, the releases of the [Ditto JavaScript client](https://github.com/eclipse/ditto-clients/tree/master/javascript) +Starting with Ditto 2.0.0, the releases of the [Ditto JavaScript client](https://github.com/eclipse-ditto/ditto-clients/tree/master/javascript) are in sync with Ditto releases. In oder to have a simplified usage of the JS client, the "api" module must no longer be explicitly imported, simply directly import one of the following 2 npm modules: @@ -215,7 +215,7 @@ simply directly import one of the following 2 npm modules: ### New features -#### [Merge/PATCH updates of digital twins](https://github.com/eclipse/ditto/issues/288) +#### [Merge/PATCH updates of digital twins](https://github.com/eclipse-ditto/ditto/issues/288) This new feature allows updating parts of a thing without affecting existing parts. You may now for example update an attribute, add a new property to a feature and delete a property of a different feature in a _single request_. The new @@ -223,57 +223,57 @@ merge functionality is available via the HTTP API and the all channels using the [Merge updates via HTTP](httpapi-concepts.html#merge-updates) or the [Merge protocol specification](protocol-specification-things-merge.html) for more details and examples. -#### [Configurable OpenID Connect / OAuth2.0 claim extraction](https://github.com/eclipse/ditto/issues/512) +#### [Configurable OpenID Connect / OAuth2.0 claim extraction](https://github.com/eclipse-ditto/ditto/issues/512) OpenID Connect support has been extended; Previously, only the `sub` field from a JWT was injected as an authorization subject. This is now configurable: The Ditto Gateway config takes a list of placeholder strings that are used to construct authorization subjects. See [OpenID Connect](installation-operating.html#openid-connect) -#### [Establishing connections to endpoints via SSH tunnel](https://github.com/eclipse/ditto/issues/985) +#### [Establishing connections to endpoints via SSH tunnel](https://github.com/eclipse-ditto/ditto/issues/985) Add support for connecting to an external system from Ditto via an SSH tunnel. -#### [DevOps API to retrieve all known connections](https://github.com/eclipse/ditto/issues/605) +#### [DevOps API to retrieve all known connections](https://github.com/eclipse-ditto/ditto/issues/605) Adds a new [DevOps command](connectivity-manage-connections.html) to list all configured, non-deleted connections. -#### [Expiring policy subjects](https://github.com/eclipse/ditto/issues/890) +#### [Expiring policy subjects](https://github.com/eclipse-ditto/ditto/issues/890) In order to give access for a certain "authorized subject" only until a fixed timestamp, a Policy subject can optionally be provided with an ["expiry" timestamp](basic-policy.html#expiring-policy-subjects) (being an ISO-8601 string). -#### [Publishing of announcement message prior to policy expiry](https://github.com/eclipse/ditto/issues/964) +#### [Publishing of announcement message prior to policy expiry](https://github.com/eclipse-ditto/ditto/issues/964) For "expiring" policy subjects it is useful to get an [announcement](basic-signals-announcement.html) message prior to the actual expiry in order to be able to prolong the temporary access rights. -#### [Addition of policy actions in order to inject a policy subject](https://github.com/eclipse/ditto/issues/926) +#### [Addition of policy actions in order to inject a policy subject](https://github.com/eclipse-ditto/ditto/issues/926) New [policy HTTP API](basic-policy.html#action-activatetokenintegration) to inject authorization subjects based on the JWT of the HTTP request. -#### [Built-in acknowledgement for search updates / strong consistency of the search index](https://github.com/eclipse/ditto/issues/914) +#### [Built-in acknowledgement for search updates / strong consistency of the search index](https://github.com/eclipse-ditto/ditto/issues/914) Ditto's search index is only eventually consistent. Applications that rely on search to for twin interactions which need to know when a change is reflected in the search index, may request the new built-in [`"search-persisted"`](basic-acknowledgements.html#built-in-acknowledgement-labels) acknowledgement label. -#### [Restoring active connection faster after a hard restart of the Ditto cluster](https://github.com/eclipse/ditto/pull/1018) +#### [Restoring active connection faster after a hard restart of the Ditto cluster](https://github.com/eclipse-ditto/ditto/pull/1018) Prioritize very active [connections](basic-connections.html) over inactive connections for reconnecting: The higher the priority, the earlier it will be reconnected on startup. -#### [Support for "Last Will" for MQTT connections](https://github.com/eclipse/ditto/issues/1021) +#### [Support for "Last Will" for MQTT connections](https://github.com/eclipse-ditto/ditto/issues/1021) Adds "Last Will" support for managed MQTT connections -#### [Allow setting retain flag for MQTT connections](https://github.com/eclipse/ditto/issues/1029) +#### [Allow setting retain flag for MQTT connections](https://github.com/eclipse-ditto/ditto/issues/1029) The `retain` flag of MQTT messages published via a managed connection is set according to a message header. -#### [Provide JWT tokens to Websocket endpoint with browser APIs](https://github.com/eclipse/ditto/issues/667) +#### [Provide JWT tokens to Websocket endpoint with browser APIs](https://github.com/eclipse-ditto/ditto/issues/667) Prior to Ditto 2.0 it was only possible to pass a JWT to the `/ws` endpoint with the `Authorization` header. As this however is not possible to influence in the browser based JavaScript API of `WebSocket`, it was not possible @@ -286,24 +286,24 @@ This is now possible by supplying the JWT via a [query-parameter `access_token`] Several bugs in Ditto 1.5.x were fixed for 2.0.0. This is a complete list of the -* [merged pull requests for milestone 2.0.0-M1](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.0.0-M1) -* [merged pull requests for milestone 2.0.0-M2](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.0.0-M2) -* [merged pull requests for milestone 2.0.0](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.0.0) +* [merged pull requests for milestone 2.0.0-M1](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.0.0-M1) +* [merged pull requests for milestone 2.0.0-M2](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.0.0-M2) +* [merged pull requests for milestone 2.0.0](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.0.0) -Here as well for the Ditto Java Client: [merged pull requests for milestone 2.0.0](https://github.com/eclipse/ditto-clients/pulls?q=is:pr+milestone:2.0.0) +Here as well for the Ditto Java Client: [merged pull requests for milestone 2.0.0](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is:pr+milestone:2.0.0) -#### ["content-type" of a Ditto Protocol JSON message did not describe its "value"](https://github.com/eclipse/ditto/pull/987) +#### ["content-type" of a Ditto Protocol JSON message did not describe its "value"](https://github.com/eclipse-ditto/ditto/pull/987) The `"content-type"` field in [Ditto Protocol headers](protocol-specification.html#headers) was intended to identify the type of the [`"value"`](protocol-specification.html#value). This was not consequently ensured which has now been fixed. -#### [Password encoding/decoding for AMQP 1.0 connections with special characters](https://github.com/eclipse/ditto/pull/996) +#### [Password encoding/decoding for AMQP 1.0 connections with special characters](https://github.com/eclipse-ditto/ditto/pull/996) When passwords contained a `+` sign, they were wrongly decoded for [AMQP 1.0 connections](connectivity-protocol-bindings-amqp10.html). -#### [Merging "extraFields" into thing payload when using "normalization" mapper](https://github.com/eclipse/ditto/issues/947) +#### [Merging "extraFields" into thing payload when using "normalization" mapper](https://github.com/eclipse-ditto/ditto/issues/947) When selecting [extra](basic-connections.html#target-topics-and-enrichment) via "enrichment", the actual value of an event could be overwritten by the "extra" data. The event data now always has priority. @@ -331,7 +331,7 @@ In order to migrate existing [things](basic-thing.html) from API version 1 to AP ### "content-type" header mapping in connection targets Due to the -[removed default content-type header mapping for connection targets](https://github.com/eclipse/ditto/pull/934), +[removed default content-type header mapping for connection targets](https://github.com/eclipse-ditto/ditto/pull/934), it might be necessary to update the way connection targets are created in case you create connection targets without explicit `headerMapping` and rely on a specific content-type on the receiving side. The request to create connection targets can be updated to contain the "old" default in this case: @@ -361,7 +361,7 @@ targets can be updated to contain the "old" default in this case: ### OpenID Connect configuration for gateway The oauth configuration section of the Gateway service has been altered to support -[arbitrary claims for authorization subjects](https://github.com/eclipse/ditto/issues/512). +[arbitrary claims for authorization subjects](https://github.com/eclipse-ditto/ditto/issues/512). The `openid-connect-issuers` map now takes key-object pairs rather than key-string pairs: old: @@ -420,5 +420,5 @@ This call returns a `CompletionStage` which finally resolves to a connected `Dit Looking forward, the current plans for Ditto 2.1.0 are: -* [Support for consuming messages from Apache Kafka](https://github.com/eclipse/ditto/issues/586) -* [Let policies import other policies to enable re-use when securing things](https://github.com/eclipse/ditto/issues/298) +* [Support for consuming messages from Apache Kafka](https://github.com/eclipse-ditto/ditto/issues/586) +* [Let policies import other policies to enable re-use when securing things](https://github.com/eclipse-ditto/ditto/issues/298) diff --git a/documentation/src/main/resources/pages/ditto/release_notes_201.md b/documentation/src/main/resources/pages/ditto/release_notes_201.md index e5ffa831021..c578eb78a57 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_201.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_201.md @@ -16,28 +16,28 @@ Compared to the latest release [2.0.0](release_notes_200.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.0.1), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.0.1), including the fixed bugs. -#### [Fixed that error responses in WS contained wrong topic path](https://github.com/eclipse/ditto/pull/1057) +#### [Fixed that error responses in WS contained wrong topic path](https://github.com/eclipse-ditto/ditto/pull/1057) The Ditto WebSocket returned a wrong `topic` for DittoProtocol messages for error responses. -#### [Optimized MQTT protocol level acknowledgements](https://github.com/eclipse/ditto/pull/1064) +#### [Optimized MQTT protocol level acknowledgements](https://github.com/eclipse-ditto/ditto/pull/1064) When using MQTT option `"reconnectForRedelivery"`, the downtime during the reconnect was optimized to be very small in order to lose only few "QoS 0" messages. -#### [Made AckUpdater work with ddata sharding](https://github.com/eclipse/ditto/pull/1063) +#### [Made AckUpdater work with ddata sharding](https://github.com/eclipse-ditto/ditto/pull/1063) "Weak" Acknowledgements were broken in 2.0.0 when Ditto was operated in a cluster with more than 1 instances. -#### [Fixed write-concern for commands requesting "search-persisted" ACK](https://github.com/eclipse/ditto/pull/1059) +#### [Fixed write-concern for commands requesting "search-persisted" ACK](https://github.com/eclipse-ditto/ditto/pull/1059) The Ditto search was updated with a wrong write concern which caused higher search update times. In addition, requests with "search-persisted" acknowledgement used the wrong write concern as well which could have caused search inconsistencies. -#### [Fixed that logging was not configurable](https://github.com/eclipse/ditto/pull/1066) +#### [Fixed that logging was not configurable](https://github.com/eclipse-ditto/ditto/pull/1066) Previously, there were no options to configure logging for Ditto - this was fixed and it is possible to either configure a "logstash" endpoint or files based log appending. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_210.md b/documentation/src/main/resources/pages/ditto/release_notes_210.md index b985ddbc70b..945b871e2ae 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_210.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_210.md @@ -7,7 +7,7 @@ summary: "Version 2.1.0 of Eclipse Ditto, released on 27.09.2021" permalink: release_notes_210.html --- -Ditto **2.1.0** is API and [binary compatible](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +Ditto **2.1.0** is API and [binary compatible](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) to prior Eclipse Ditto 2.x versions. ## Changelog @@ -41,10 +41,10 @@ The following non-functional work is also included:
For a complete list of all merged PRs, inspect the following milestones: -* [merged pull requests for milestone 2.1.0-M1](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.1.0-M1) -* [merged pull requests for milestone 2.1.0-M2](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.1.0-M2) -* [merged pull requests for milestone 2.1.0-M3](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.1.0-M3) -* [merged pull requests for milestone 2.1.0](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.1.0) +* [merged pull requests for milestone 2.1.0-M1](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.1.0-M1) +* [merged pull requests for milestone 2.1.0-M2](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.1.0-M2) +* [merged pull requests for milestone 2.1.0-M3](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.1.0-M3) +* [merged pull requests for milestone 2.1.0](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.1.0)

@@ -54,7 +54,7 @@ bugfixes were added. ### Changes -#### [Support using URLs in "definitions"](https://github.com/eclipse/ditto/pull/1185) +#### [Support using URLs in "definitions"](https://github.com/eclipse-ditto/ditto/pull/1185) In addition to using the currently supported pattern `::` in [thing definitions](basic-thing.html#definition) and @@ -64,9 +64,9 @@ This is useful to link a Thing to a model definition available via an HTTP URL, [WoT (Web of Things) Thing Model](https://www.w3.org/TR/wot-thing-description11/#thing-model). A discussion of how a WoT "Thing Model" can be mapped to an Eclipse Ditto "Thing" is currently -[ongoing on GitHub](https://github.com/eclipse/ditto/discussions/1163), please join in to provide feedback. +[ongoing on GitHub](https://github.com/eclipse-ditto/ditto/discussions/1163), please join in to provide feedback. -#### [Addition of category "misconfigured" for current connection status](https://github.com/eclipse/ditto/pull/1151) +#### [Addition of category "misconfigured" for current connection status](https://github.com/eclipse-ditto/ditto/pull/1151) When [retrieving the connection status](connectivity-manage-connections.html#retrieve-connection-status), a connection was previously in status `"failed"` when e.g. the credentials to an endpoint were wrong or a queue in a message broker @@ -74,7 +74,7 @@ was not existing. By adding a new status `"misconfigured"` such configuration errors are now differentiable from real "failures". -#### [Tracing support, reporting traces to an "Open Telemetry" endpoint](https://github.com/eclipse/ditto/issues/1135) +#### [Tracing support, reporting traces to an "Open Telemetry" endpoint](https://github.com/eclipse-ditto/ditto/issues/1135) Eclipse Ditto can now optionally be configured to enable [tracing](installation-operating.html#tracing) and to report traces to an "Open Telemetry" endpoint. @@ -90,18 +90,18 @@ to end users. E.g. support configuration of a [logstash endpoint or a custom file appender](installation-operating.html#logging). -#### [Improving background deletion of dangling DB journal entries / snapshots based on the current MongoDB load](https://github.com/eclipse/ditto/pull/1168) +#### [Improving background deletion of dangling DB journal entries / snapshots based on the current MongoDB load](https://github.com/eclipse-ditto/ditto/pull/1168) The deletion of dangling / no longer needed DB entries was improved to react better to the current load of the database. -#### [Improving search update by applying "delta updates" saving lots of bandwidth to MongoDB](https://github.com/eclipse/ditto/pull/1174) +#### [Improving search update by applying "delta updates" saving lots of bandwidth to MongoDB](https://github.com/eclipse-ditto/ditto/pull/1174) Previous versions of Ditto updated the documents in the search index always completely, also if only one value of a thing was modified. This is now drastically improved so that only changed values are updated in the search index which saves a lot of network traffic towards the database. -#### [Reducing cluster communication for search updates using a smart cache](https://github.com/eclipse/ditto/pull/1150) +#### [Reducing cluster communication for search updates using a smart cache](https://github.com/eclipse-ditto/ditto/pull/1150) Minimized internal cluster roundtrips to load thing data in order to calculate the search index documents by using a cache. @@ -109,7 +109,7 @@ cache. ### New features -#### [Support consuming messages from Apache Kafka](https://github.com/eclipse/ditto/issues/586) +#### [Support consuming messages from Apache Kafka](https://github.com/eclipse-ditto/ditto/issues/586) Implemented connection sources for Apache Kafka connections. This latest addition completes the Apache Kafka integration in Eclipse Ditto and can utilize Ditto's @@ -122,7 +122,7 @@ integrate your favorite Digital Twin framework with your favorite event streamin When updating from a Ditto version where Kafka was used before, please notice and follow the [migration notes](#adapt-kafka-client-configuration) on Kafka related configuration changes. -#### [Conditional requests (updates + retrieves)](https://github.com/eclipse/ditto/issues/559) +#### [Conditional requests (updates + retrieves)](https://github.com/eclipse-ditto/ditto/issues/559) Utilizing the existing [RQL](basic-rql.html) filters, it is now possible to specify a `condition` when performing any of operations (like modifications or retrievals) on a thing. @@ -134,30 +134,30 @@ older than a specified timestamp in the condition. Please have a look at our [blog post](2021-09-23-conditional-requests.html) about this new feature and the newly added [documentation of conditions](basic-connections.html) to find out more. -#### [Enrichment of "ThingDeleted" events with extra fields](https://github.com/eclipse/ditto/pull/1184) +#### [Enrichment of "ThingDeleted" events with extra fields](https://github.com/eclipse-ditto/ditto/pull/1184) Previously, Eclipse Ditto could not [enrich](basic-enrichment.html) `ThingDeleted` events with additional data from the deleted thing - this now is supported. -#### [HMAC based authentication for Ditto managed connections](https://github.com/eclipse/ditto/issues/1060) +#### [HMAC based authentication for Ditto managed connections](https://github.com/eclipse-ditto/ditto/issues/1060) For better integration with Microsoft's Azure and Amazon's AWS ecosystems, Eclipse Ditto HTTP connections now support HMAC-based authentication methods such as that of "Azure Monitor Data Collector" and "AWS Signature Version 4". Please read the [blog post](2021-06-17-hmac-credentials.html) to learn more about that. -#### [SASL authentication for Azure IoT Hub](https://github.com/eclipse/ditto/issues/1078) +#### [SASL authentication for Azure IoT Hub](https://github.com/eclipse-ditto/ditto/issues/1078) For better integration with "Azure IoT Hub", Eclipse Ditto AMQP and HTTP connections now support its shared access signature authentication. -#### [Publishing of connection opened/closed announcements](https://github.com/eclipse/ditto/issues/1052) +#### [Publishing of connection opened/closed announcements](https://github.com/eclipse-ditto/ditto/issues/1052) An Eclipse Ditto connection can now be configured to send an [announcement](protocol-specification-connections-announcement.html) to the target endpoint, when the connection is opening and when it is closing again (e.g. during a restart of Eclipse Ditto). -#### [Support "at least once" delivery for policy subject deletion announcements](https://github.com/eclipse/ditto/issues/1107) +#### [Support "at least once" delivery for policy subject deletion announcements](https://github.com/eclipse-ditto/ditto/issues/1107) Policy subject [deletion announcements](basic-policy.html#subject-deletion-announcements) can now be configured to be delivered "at least once", using Eclipse Ditto's built in [acknowledgement](basic-acknowledgements.html) mechanism. @@ -167,14 +167,14 @@ delivered "at least once", using Eclipse Ditto's built in [acknowledgement](basi Several bugs in Ditto 2.0.x were fixed for 2.1.0. This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.1.0), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.1.0), including the fixed bugs. -#### [Fix "search-persisted" acknowledgement not working for thing deletion](https://github.com/eclipse/ditto/pull/1141) +#### [Fix "search-persisted" acknowledgement not working for thing deletion](https://github.com/eclipse-ditto/ditto/pull/1141) Previously, when requesting the built-in [acknowledgement label "search-persisted"](basic-acknowledgements.html#built-in-acknowledgement-labels) for thing deletions the response did not wait for the deletion in the search index which has been fixed now. -#### [Fix reconnect loop to MQTT brokers when using separate MQTT publisher client](https://github.com/eclipse/ditto/pull/1117) +#### [Fix reconnect loop to MQTT brokers when using separate MQTT publisher client](https://github.com/eclipse-ditto/ditto/pull/1117) Eclipse Ditto 2.0.0 could run into an endless reconnect loop to an MQTT broker when a separate publisher client was configured. This has been fixed. @@ -214,14 +214,14 @@ ditto.connectivity.connection { ## Ditto clients For a complete list of all merged client PRs, inspect the following milestones: -* [merged pull requests for milestone 2.1.0-M1](https://github.com/eclipse/ditto-clients/pulls?q=is:pr+milestone:2.1.0-M1) -* [merged pull requests for milestone 2.1.0-M2](https://github.com/eclipse/ditto-clients/pulls?q=is:pr+milestone:2.1.0-M2) -* [merged pull requests for milestone 2.1.0-M3](https://github.com/eclipse/ditto-clients/pulls?q=is:pr+milestone:2.1.0-M3) -* [merged pull requests for milestone 2.1.0](https://github.com/eclipse/ditto-clients/pulls?q=is:pr+milestone:2.1.0) +* [merged pull requests for milestone 2.1.0-M1](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is:pr+milestone:2.1.0-M1) +* [merged pull requests for milestone 2.1.0-M2](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is:pr+milestone:2.1.0-M2) +* [merged pull requests for milestone 2.1.0-M3](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is:pr+milestone:2.1.0-M3) +* [merged pull requests for milestone 2.1.0](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is:pr+milestone:2.1.0) ### Ditto Java SDK -#### [Enhance Ditto java client with a "disconnection listener"](https://github.com/eclipse/ditto-clients/pull/167) +#### [Enhance Ditto java client with a "disconnection listener"](https://github.com/eclipse-ditto/ditto-clients/pull/167) A new disconnection listener can be configured which: * is called whenever the connection to the Ditto backend was disconnected @@ -230,15 +230,15 @@ A new disconnection listener can be configured which: ### Ditto JavaScript SDK -See separate [Changelog](https://github.com/eclipse/ditto-clients/blob/master/javascript/CHANGELOG.md) of JS client. +See separate [Changelog](https://github.com/eclipse-ditto/ditto-clients/blob/master/javascript/CHANGELOG.md) of JS client. ## Roadmap Looking forward, the current plans for Ditto 2.2.0 are: -* [Add HTTP API for "live" commands](https://github.com/eclipse/ditto/issues/106) -* [Add support for twin life-cycle events](https://github.com/eclipse/ditto/issues/898) +* [Add HTTP API for "live" commands](https://github.com/eclipse-ditto/ditto/issues/106) +* [Add support for twin life-cycle events](https://github.com/eclipse-ditto/ditto/issues/898) * Defining "channel" conditions on API requests against Ditto so that Ditto decides based on the condition whether to use "twin" or "live" channel * Concept and first work on a WoT (Web of Things) integration diff --git a/documentation/src/main/resources/pages/ditto/release_notes_211.md b/documentation/src/main/resources/pages/ditto/release_notes_211.md index 7287d5199e4..5367a180a4e 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_211.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_211.md @@ -16,35 +16,35 @@ Compared to the latest release [2.1.0](release_notes_210.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.1.1), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.1.1), including the fixed bugs. In addition, this is a complete list of the -[merged Ditto Client pull requests](https://github.com/eclipse/ditto-clients/pulls?q=is%3Apr+milestone%3A2.1.1), including the fixed bugs. +[merged Ditto Client pull requests](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is%3Apr+milestone%3A2.1.1), including the fixed bugs. -#### [Stabilize connection live status for AMQP 1.0 connections](https://github.com/eclipse/ditto/pull/1206) +#### [Stabilize connection live status for AMQP 1.0 connections](https://github.com/eclipse-ditto/ditto/pull/1206) The connection live status was not always correct for AMQP 1.0 connections which were `misconfigured`. -#### [Fixed drop behavior of mapping queue in LegacyBaseConsumerActor](https://github.com/eclipse/ditto/pull/1195) +#### [Fixed drop behavior of mapping queue in LegacyBaseConsumerActor](https://github.com/eclipse-ditto/ditto/pull/1195) Connections using the `LegacyBaseConsumerActor` like AMQP 0.9.1, MQTT, HTTP did silently drop messages if the configured buffer size was reached and backpressure was applied. This is now logged as warning and a retry is invoked. -#### [Fix status 500 when sorting a field containing non-primitive values](https://github.com/eclipse/ditto/pull/1207) +#### [Fix status 500 when sorting a field containing non-primitive values](https://github.com/eclipse-ditto/ditto/pull/1207) When sorting on e.g. JSON objects in the things-search a HTTP status 500 was returned. -#### [Explicitly configure MongoDB query batchSize same as the limit](https://github.com/eclipse/ditto/pull/1211) +#### [Explicitly configure MongoDB query batchSize same as the limit](https://github.com/eclipse-ditto/ditto/pull/1211) The used MongoDB driver used a max `batchSize` of `16` for performing searches. When the requested result size was higher than 16, this lead to multiple roundtrips/batches for a search resulting in not ideal performance. -#### [Fix mqtt connection status for sources with multiple addresses](https://github.com/eclipse/ditto/pull/1214) +#### [Fix mqtt connection status for sources with multiple addresses](https://github.com/eclipse-ditto/ditto/pull/1214) The connection live status for MQTT connections with sources containing multiple addresses contained `failure` status entries for "missing" sources. -#### Ditto Java Client: [Shutdown executor after stream cancellation](https://github.com/eclipse/ditto-clients/pull/174) +#### Ditto Java Client: [Shutdown executor after stream cancellation](https://github.com/eclipse-ditto/ditto-clients/pull/174) The Ditto Java client had a thread leak which is now fixed. \ No newline at end of file diff --git a/documentation/src/main/resources/pages/ditto/release_notes_212.md b/documentation/src/main/resources/pages/ditto/release_notes_212.md index 2f16cbd148d..e5819b1171b 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_212.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_212.md @@ -16,25 +16,25 @@ Compared to the latest release [2.1.1](release_notes_211.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.1.2), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.1.2), including the fixed bugs. -#### [Improve performance of JWT validations by caching jwt validator](https://github.com/eclipse/ditto/pull/1217) +#### [Improve performance of JWT validations by caching jwt validator](https://github.com/eclipse-ditto/ditto/pull/1217) For some users of Ditto, the performance of JWT validations was dramatically bad. This bugfix caches the creation of the JWT parser based on the public key and should dramatically improve the performance of HTTP requests using JWT. -#### [Improve Kafka consumer performance](https://github.com/eclipse/ditto/pull/1218) +#### [Improve Kafka consumer performance](https://github.com/eclipse-ditto/ditto/pull/1218) This bugfix should reduce the CPU load in the connectivity service and number of requests/second to the Kafka broker by increasing `fetch.max.wait.ms` to 5 seconds. It should also reduce the consumer lag due to a lack of threads when there are a lot of consumers running. -#### [Keep order of json elements in connection model in set structures](https://github.com/eclipse/ditto/pull/1219) +#### [Keep order of json elements in connection model in set structures](https://github.com/eclipse-ditto/ditto/pull/1219) Previously, the JSON element order e.g. in arrays in a managed `connection` could be mixed up, e.g. from creation to persistence. This has been fixed by maintaining the JSON element order in the connection model. -#### [Updated to Akka HTTP 10.2.7 due to critical reported CVE](https://github.com/eclipse/ditto/pull/1222) +#### [Updated to Akka HTTP 10.2.7 due to critical reported CVE](https://github.com/eclipse-ditto/ditto/pull/1222) The for Ditto's HTTP API used library contained a critical security issue [CVE-2021-42697](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-42697) which has been resolved in Akka HTTP 10.2.7 diff --git a/documentation/src/main/resources/pages/ditto/release_notes_213.md b/documentation/src/main/resources/pages/ditto/release_notes_213.md index 697692848f3..303f903dc8b 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_213.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_213.md @@ -16,9 +16,9 @@ Compared to the latest release [2.1.2](release_notes_212.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.1.3), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.1.3), including the fixed bugs. -#### [Update logback to 1.2.8 due to "possibility of vulnerability"](https://github.com/eclipse/ditto/pull/1253) +#### [Update logback to 1.2.8 due to "possibility of vulnerability"](https://github.com/eclipse-ditto/ditto/pull/1253) The reported [LOGBACK-1591](https://jira.qos.ch/browse/LOGBACK-1591) reports a "Possibility of vulnerability" with a medium severity. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_220.md b/documentation/src/main/resources/pages/ditto/release_notes_220.md index d28f48f3c95..b43ac979c20 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_220.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_220.md @@ -7,7 +7,7 @@ summary: "Version 2.2.0 of Eclipse Ditto, released on 22.11.2021" permalink: release_notes_220.html --- -Ditto **2.2.0** is API and [binary compatible](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +Ditto **2.2.0** is API and [binary compatible](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) to prior Eclipse Ditto 2.x versions. ## Changelog @@ -33,7 +33,7 @@ The following non-functional work is also included:
For a complete list of all merged PRs, inspect the following milestones: -* [merged pull requests for milestone 2.2.0](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.2.0) +* [merged pull requests for milestone 2.2.0](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.2.0)

@@ -43,7 +43,7 @@ bugfixes were added. ### Changes -#### [Allow using the dash `-` as part of the "namespace" part in Ditto thing and policy IDs](https://github.com/eclipse/ditto/issues/1231) +#### [Allow using the dash `-` as part of the "namespace" part in Ditto thing and policy IDs](https://github.com/eclipse-ditto/ditto/issues/1231) A feedback from our community was that the dash `-` should be an allowed character in the [namespace](basic-namespaces-and-names.html#namespace) part of Ditto managed IDs. @@ -53,7 +53,7 @@ to use the reverse-domain-notation for a package name with domains including das ### New features -#### [Filter for twin life-cycle events](https://github.com/eclipse/ditto/issues/898) +#### [Filter for twin life-cycle events](https://github.com/eclipse-ditto/ditto/issues/898) When applying [filtering for change notifications](basic-changenotifications.html#filtering), the existing RQL based filter was enhanced with the possibility to use [topic and resource placeholders](basic-rql.html#placeholders-as-query-properties) @@ -64,17 +64,17 @@ Only emit events for Thing creation and deletion: and(in(topic:action,'created','deleted'),eq(resource:path,'/')) ``` -#### [Possibility to forward connection logs](https://github.com/eclipse/ditto/pull/1230) +#### [Possibility to forward connection logs](https://github.com/eclipse-ditto/ditto/pull/1230) Configure [log publishing](connectivity-manage-connections.html#publishing-connection-logs)for your Ditto managed connections in order to get connection logs wherever you need them to analyze. -#### [Add OAuth2 client credentials flow as an authentication mechanism for Ditto managed HTTP connections](https://github.com/eclipse/ditto/pull/1233) +#### [Add OAuth2 client credentials flow as an authentication mechanism for Ditto managed HTTP connections](https://github.com/eclipse-ditto/ditto/pull/1233) Have a look at our [blog post](2021-11-03-oauth2.html) which shares an example of how to configure a Ditto managed HTTP connection to make use of OAuth2.0 authentication. -#### [Enable loading additional extra JavaScript libraries for Rhino based JS mapping engine](https://github.com/eclipse/ditto/pull/1208) +#### [Enable loading additional extra JavaScript libraries for Rhino based JS mapping engine](https://github.com/eclipse-ditto/ditto/pull/1208) The used [Rhino JS engine](https://github.com/mozilla/rhino) allows making use of "CommonJS" in order to load JS modules via `require('')` into the engine. This feature has now been exposed by Ditto, configuring the environment @@ -87,7 +87,7 @@ additional JS modules into the container. Several bugs in Ditto 2.1.x were fixed for 2.2.0. This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.2.0), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.2.0), including the fixed bugs. ## Migration notes @@ -97,7 +97,7 @@ No migrations required updating from Ditto 2.1.x ## Ditto clients For a complete list of all merged client PRs, inspect the following milestones: -* [merged pull requests for milestone 2.2.0](https://github.com/eclipse/ditto-clients/pulls?q=is:pr+milestone:2.2.0) +* [merged pull requests for milestone 2.2.0](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is:pr+milestone:2.2.0) ### Ditto Java SDK @@ -105,13 +105,13 @@ No mentionable changes/enhancements/bugfixes. ### Ditto JavaScript SDK -See separate [Changelog](https://github.com/eclipse/ditto-clients/blob/master/javascript/CHANGELOG.md) of JS client. +See separate [Changelog](https://github.com/eclipse-ditto/ditto-clients/blob/master/javascript/CHANGELOG.md) of JS client. ## Roadmap Looking forward, the current plans for Ditto 2.3.0 are: -* [Add HTTP API for "live" commands](https://github.com/eclipse/ditto/issues/106) -* [Smart channel strategy for live/twin read access](https://github.com/eclipse/ditto/issues/1228) +* [Add HTTP API for "live" commands](https://github.com/eclipse-ditto/ditto/issues/106) +* [Smart channel strategy for live/twin read access](https://github.com/eclipse-ditto/ditto/issues/1228) * More work on concept and first work on a WoT (Web of Things) integration diff --git a/documentation/src/main/resources/pages/ditto/release_notes_221.md b/documentation/src/main/resources/pages/ditto/release_notes_221.md index 76b4d49992f..43d6f66eea6 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_221.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_221.md @@ -16,20 +16,20 @@ Compared to the latest release [2.2.0](release_notes_220.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.2.1), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.2.1), including the fixed bugs. -#### [Update logback to 1.2.8 due to "possibility of vulnerability"](https://github.com/eclipse/ditto/pull/1253) +#### [Update logback to 1.2.8 due to "possibility of vulnerability"](https://github.com/eclipse-ditto/ditto/pull/1253) The reported [LOGBACK-1591](https://jira.qos.ch/browse/LOGBACK-1591) reports a "Possibility of vulnerability" with a medium severity. -#### [Switch to ByteSerializer and ByteDeserializer for Kafka Consumer and Publisher](https://github.com/eclipse/ditto/pull/1241) +#### [Switch to ByteSerializer and ByteDeserializer for Kafka Consumer and Publisher](https://github.com/eclipse-ditto/ditto/pull/1241) With Ditto 2.2.0, when consuming binary messages from Apache Kafka, the charset was not considered correctly and therefore binary payload (e.g. protobuf messages) were not consumed correctly. That was fixed by using the binary deserializer. -#### [Also disable hostname verification when HTTP connection wants to ignore SSL](https://github.com/eclipse/ditto/pull/1243) +#### [Also disable hostname verification when HTTP connection wants to ignore SSL](https://github.com/eclipse-ditto/ditto/pull/1243) For [managed HTTP connections](connectivity-protocol-bindings-http.html) for which `validateCertificates` was disabled, single HTTP interactions when publishing messages were still using certificate validation. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_230.md b/documentation/src/main/resources/pages/ditto/release_notes_230.md index 101879640da..15b91da08a4 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_230.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_230.md @@ -7,7 +7,7 @@ summary: "Version 2.3.0 of Eclipse Ditto, released on 21.01.2022" permalink: release_notes_230.html --- -Ditto **2.3.0** is API and [binary compatible](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +Ditto **2.3.0** is API and [binary compatible](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) to prior Eclipse Ditto 2.x versions. ## Changelog @@ -49,7 +49,7 @@ The following non-functional work is also included:
For a complete list of all merged PRs, inspect the following milestones: -* [merged pull requests for milestone 2.3.0](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.3.0) +* [merged pull requests for milestone 2.3.0](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.3.0)

@@ -60,7 +60,7 @@ bugfixes were added. ### New features -#### [HTTP API for "live" commands](https://github.com/eclipse/ditto/issues/106) +#### [HTTP API for "live" commands](https://github.com/eclipse-ditto/ditto/issues/106) Ditto's ["live" channel](protocol-twinlive.html#live) is now also available for commands invoked via HTTP API. See also the [blogpost covering that topic](2021-12-20-http-live-channel.html). @@ -72,7 +72,7 @@ Live commands bypass the twin and go directly to the devices. However, an existing twin is still required for policy enforcement. -#### [Smart channel strategy for live/twin read access](https://github.com/eclipse/ditto/issues/1228) +#### [Smart channel strategy for live/twin read access](https://github.com/eclipse-ditto/ditto/issues/1228) Ditto adds support for selecting the "twin" or the "live" channel for thing query commands based on an [RQL condition](basic-rql.html) of a newly added parameter @@ -82,12 +82,12 @@ See also the [blogpost covering that topic](2021-12-22-live-channel-condition.ht In addition, a new [payload mapper](connectivity-mapping.html#updatetwinwithliveresponse-mapper) automatically updating the twin based on received live data from devices was added. -#### [Configurable allowing creation of entities based on namespace and authenticated subjects](https://github.com/eclipse/ditto/pull/1251) +#### [Configurable allowing creation of entities based on namespace and authenticated subjects](https://github.com/eclipse-ditto/ditto/pull/1251) This added feature allows configuring restrictions, which [authenticated subjects](basic-auth.html#authenticated-subjects) may create new entities (things / policies) in which namespaces. -#### [Allow using `*` as a placeholder for the feature id in selected fields](https://github.com/eclipse/ditto/pull/1277) +#### [Allow using `*` as a placeholder for the feature id in selected fields](https://github.com/eclipse-ditto/ditto/pull/1277) When selecting for certain [`fields` of a thing](httpapi-concepts.html#field-selector-with-wildcard) or when using [signal enrichment (extraFields)](basic-enrichment.html) in order to add more (unchanged) data from a twin to e.g. events @@ -98,7 +98,7 @@ the wildcard `*` can now be used in order to select all features of a thing with Several bugs in Ditto 2.2.x were fixed for 2.3.0. This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.3.0), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.3.0), including the fixed bugs. ## Migration notes @@ -108,7 +108,7 @@ No migrations required updating from Ditto 2.2.x ## Ditto clients For a complete list of all merged client PRs, inspect the following milestones: -* [merged pull requests for milestone 2.3.0](https://github.com/eclipse/ditto-clients/pulls?q=is:pr+milestone:2.3.0) +* [merged pull requests for milestone 2.3.0](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is:pr+milestone:2.3.0) ### Ditto Java SDK @@ -116,7 +116,7 @@ No mentionable changes/enhancements/bugfixes. ### Ditto JavaScript SDK -See separate [Changelog](https://github.com/eclipse/ditto-clients/blob/master/javascript/CHANGELOG.md) of JS client. +See separate [Changelog](https://github.com/eclipse-ditto/ditto-clients/blob/master/javascript/CHANGELOG.md) of JS client. ## Roadmap @@ -124,4 +124,4 @@ See separate [Changelog](https://github.com/eclipse/ditto-clients/blob/master/ja Looking forward, the current plans for Ditto 2.4.0 are: * Update service code to Java 17 (APIs stay at Java 8) + run Ditto containers with Java 17 runtime -* Continue work on the started [WoT (Web of Things) integration](https://github.com/eclipse/ditto/pull/1270) +* Continue work on the started [WoT (Web of Things) integration](https://github.com/eclipse-ditto/ditto/pull/1270) diff --git a/documentation/src/main/resources/pages/ditto/release_notes_231.md b/documentation/src/main/resources/pages/ditto/release_notes_231.md index 55f2ec3af1a..8031aa26f6a 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_231.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_231.md @@ -16,23 +16,23 @@ Compared to the latest release [2.3.0](release_notes_230.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.3.1), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.3.1), including the fixed bugs. -#### [Fix that placeholder `time:now` could not be used in connection header mapping](https://github.com/eclipse/ditto/pull/1292) +#### [Fix that placeholder `time:now` could not be used in connection header mapping](https://github.com/eclipse-ditto/ditto/pull/1292) The in version 2.3.0 newly added placeholder `time:now` could not be used in connectivity's header mapping, this is now fixed. -#### [Enable use of entity placeholders for MQTT and HTTP connections](https://github.com/eclipse/ditto/pull/1293) +#### [Enable use of entity placeholders for MQTT and HTTP connections](https://github.com/eclipse-ditto/ditto/pull/1293) The `entity:id` placeholder could not be used in MQTT and HTTP connections, this is now fixed. -#### [Reduce the likelihood of search index inconsistency due to reordering of patch updates](https://github.com/eclipse/ditto/pull/1296) +#### [Reduce the likelihood of search index inconsistency due to reordering of patch updates](https://github.com/eclipse-ditto/ditto/pull/1296) This fix speeds up the consistency of search entries which could get inconsistent during rolling updates for very active things. -#### [Fixed that JSON `null` in "correlation-id" of Ditto Protocol headers were parsed as JSON String "null"](https://github.com/eclipse/ditto/pull/1295) +#### [Fixed that JSON `null` in "correlation-id" of Ditto Protocol headers were parsed as JSON String "null"](https://github.com/eclipse-ditto/ditto/pull/1295) When parsing messages as Ditto Protocol, a `"correlation-id"` header field being `null` was translated to the string literal `"null"` - this has been fixed by treating the "correlation-id" as not being present instead. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_232.md b/documentation/src/main/resources/pages/ditto/release_notes_232.md index 2a5e48427e0..56105dac21f 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_232.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_232.md @@ -16,23 +16,23 @@ Compared to the latest release [2.3.1](release_notes_231.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.3.2), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.3.2), including the fixed bugs. -#### [Fix search consistency for failed patch updates](https://github.com/eclipse/ditto/pull/1300) +#### [Fix search consistency for failed patch updates](https://github.com/eclipse-ditto/ditto/pull/1300) The search index could get never consistent for some things, as failed "patch updates" to the search index were not retried. -#### [Fix search sync infinite loop](https://github.com/eclipse/ditto/pull/1301) +#### [Fix search sync infinite loop](https://github.com/eclipse-ditto/ditto/pull/1301) Forcing a full search index update could lead to an infinite loop when processing "Thing Deleted" events. -#### [Ignore DittoMessageMapper for Hono delivery failed notifications](https://github.com/eclipse/ditto/pull/1299) +#### [Ignore DittoMessageMapper for Hono delivery failed notifications](https://github.com/eclipse-ditto/ditto/pull/1299) Delivery failure notification sent via Eclipse Hono with content-type `application/vnd.eclipse-hono-delivery-failure-notification+json` are now excluded to be handled by the default `DittoMessageMapper` used in connections. -#### [Fixed potential race condition in LiveSignalEnforcement](https://github.com/eclipse/ditto/pull/1305) +#### [Fixed potential race condition in LiveSignalEnforcement](https://github.com/eclipse-ditto/ditto/pull/1305) For live response messages a race condition could happen where a 503 ("Thing not available") exception was produced instead of running in a 408 timeout. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_240.md b/documentation/src/main/resources/pages/ditto/release_notes_240.md index fd16ed09dce..81d52d3adc0 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_240.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_240.md @@ -7,7 +7,7 @@ summary: "Version 2.4.0 of Eclipse Ditto, released on 14.04.2022" permalink: release_notes_240.html --- -Ditto **2.4.0** is API and [binary compatible](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) +Ditto **2.4.0** is API and [binary compatible](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0005-semantic-versioning.md) to prior Eclipse Ditto 2.x versions. ## Changelog @@ -40,8 +40,8 @@ The following non-functional work is also included:
For a complete list of all merged PRs, inspect the following milestones: -* [merged pull requests for milestone 2.4.0-M1](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.4.0-M1) -* [merged pull requests for milestone 2.4.0](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:2.4.0) +* [merged pull requests for milestone 2.4.0-M1](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.4.0-M1) +* [merged pull requests for milestone 2.4.0](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:2.4.0)

@@ -52,7 +52,7 @@ bugfixes were added: ### Changes -#### [Upgrade to Java 17 + change of Java runtime to Hotspot](https://github.com/eclipse/ditto/issues/1283) +#### [Upgrade to Java 17 + change of Java runtime to Hotspot](https://github.com/eclipse-ditto/ditto/issues/1283) We upgraded the compiler target level for our service modules from 11 to 17 and also use a Java 17 runtime environment for our service containers. Please note that the Ditto model still remains compatible to Java 8. @@ -65,7 +65,7 @@ Docker images for the following architectures will be published to docker.io: * `linux/amd64` * `linux/arm64` (new) -#### [Removal of rate limiting / throttling limits as default](https://github.com/eclipse/ditto/pull/1324) +#### [Removal of rate limiting / throttling limits as default](https://github.com/eclipse-ditto/ditto/pull/1324) By default, Ditto had configurations in place to rate limit consumption of messages received via: * AMQP 1.0 connections @@ -78,7 +78,7 @@ and can be enabled manually, as mentioned in the [configuration - rate limiting ### New features -#### [W3C WoT (Web of Things) integration](https://github.com/eclipse/ditto/issues/1034) +#### [W3C WoT (Web of Things) integration](https://github.com/eclipse-ditto/ditto/issues/1034) Ditto adds and optional (and currently *experimental*) integration of the [Web of Things (WoT) Thing Description 1.1](https://www.w3.org/TR/wot-thing-description11/) specification. @@ -90,13 +90,13 @@ skeleton upon creation of a thing. For more details, please have a look at the [blogpost](2022-03-03-wot-integration.html) and the [WoT integration documentation](basic-wot-integration.html). -#### [SSE (ServerSentEvent) API for subscribing to messages](https://github.com/eclipse/ditto/issues/1186) +#### [SSE (ServerSentEvent) API for subscribing to messages](https://github.com/eclipse-ditto/ditto/issues/1186) Messages to or from a digital twin can now be subscribed to with the [SSE endpoint](httpapi-sse.html), either on [Thing level](httpapi-sse.html#subscribe-for-messages-for-a-specific-thing) or for a specific [Feature](httpapi-sse.html#subscribe-for-messages-of-a-specific-feature-of-a-specific-thing) -#### [Recovery status for connections indicating when e.g. recovery is no longer tried after max backoff](https://github.com/eclipse/ditto/pull/1336) +#### [Recovery status for connections indicating when e.g. recovery is no longer tried after max backoff](https://github.com/eclipse-ditto/ditto/pull/1336) The new recovery status contains one of the values: * ongoing @@ -107,12 +107,12 @@ The new recovery status contains one of the values: and can be used to find out whether an automatic failover is still ongoing or if the max amount of configured reconnects applying backoff was reached and that recovery is no longer happening. -#### [Enhance placeholders to resolve to multiple values](https://github.com/eclipse/ditto/pull/1331) +#### [Enhance placeholders to resolve to multiple values](https://github.com/eclipse-ditto/ditto/pull/1331) Placeholders may now resolve to multiple values instead of only a single one which enables e.g. applying [placeholder functions](basic-placeholders.html#function-expressions) to each element of an array. -#### [Advanced JWT placeholder operations](https://github.com/eclipse/ditto/pull/1309) +#### [Advanced JWT placeholder operations](https://github.com/eclipse-ditto/ditto/pull/1309) Using the above feature of placeholders being resolved to multiple values, the JWT placeholder, which can be used in [scope of the OpenID connect configuration](basic-placeholders.html#scope-openid-connect-configuration), can now @@ -126,7 +126,7 @@ Example extracting only subjects from a JSON array "roles" contained in a JWT en {%raw%}{{ jwt:extra/roles | fn:filter('like','*moderator') }}{%endraw%} ``` -#### [Support for a wildcard/placeholder identifying the changed feature in order to enrich e.g. its definition](https://github.com/eclipse/ditto/issues/710) +#### [Support for a wildcard/placeholder identifying the changed feature in order to enrich e.g. its definition](https://github.com/eclipse-ditto/ditto/issues/710) Using the above feature of placeholders being resolved to multiple values, it is now possible to use a placeholder `{%raw%}{{ feature:id }}{%endraw%}` as part of an [enrichment `extraFields` pointer](basic-enrichment.html) resolving @@ -143,8 +143,8 @@ published via websocket or a connection: Several bugs in Ditto 2.3.x were fixed for 2.4.0. This is a complete list of the -[merged pull requests for 2.4.0-M1](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.4.0-M1) and -[merged pull requests for 2.4.0](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.4.0), +[merged pull requests for 2.4.0-M1](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.4.0-M1) and +[merged pull requests for 2.4.0](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.4.0), including the fixed bugs. @@ -160,11 +160,11 @@ Migrations required updating from Ditto 2.3.x or earlier versions: ## Ditto clients For a complete list of all merged client PRs, inspect the following milestones: -* [merged pull requests for milestone 2.4.0-M1](https://github.com/eclipse/ditto-clients/pulls?q=is:pr+milestone:2.4.0-M1) +* [merged pull requests for milestone 2.4.0-M1](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is:pr+milestone:2.4.0-M1) ### Ditto Java SDK -#### [Fix returning the revision of a Policy when retrieved via the client](https://github.com/eclipse/ditto-clients/pull/182) +#### [Fix returning the revision of a Policy when retrieved via the client](https://github.com/eclipse-ditto/ditto-clients/pull/182) When using the `client.policies().retrievePolicy(PolicyId)` functionality in the Ditto Java client, the `getRevision()` method of the returned policy was always empty. @@ -172,7 +172,7 @@ The revision will now be included. ### Ditto JavaScript SDK -See separate [Changelog](https://github.com/eclipse/ditto-clients/blob/master/javascript/CHANGELOG.md) of JS client. +See separate [Changelog](https://github.com/eclipse-ditto/ditto-clients/blob/master/javascript/CHANGELOG.md) of JS client. ## Roadmap diff --git a/documentation/src/main/resources/pages/ditto/release_notes_241.md b/documentation/src/main/resources/pages/ditto/release_notes_241.md index 3d00c17865f..52439f04927 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_241.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_241.md @@ -16,14 +16,14 @@ Compared to the latest release [2.4.0](release_notes_240.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.4.1), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.4.1), including the fixed bugs. -#### [In case of ThingDeleteModel always consider model as not outdated](https://github.com/eclipse/ditto/pull/1368) +#### [In case of ThingDeleteModel always consider model as not outdated](https://github.com/eclipse-ditto/ditto/pull/1368) The search index could run into consistency problem which this fix improves. The final fix for having a fully consistent search index however will only be available in Ditto `3.0.0` where the index was completely rebuilt. -#### [Enable self signed certificates for kafka](https://github.com/eclipse/ditto/pull/1456) +#### [Enable self signed certificates for kafka](https://github.com/eclipse-ditto/ditto/pull/1456) Fix that self-signed certificates for Kafka connections could not be used. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_242.md b/documentation/src/main/resources/pages/ditto/release_notes_242.md index d32a2343d42..a3112c3a125 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_242.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_242.md @@ -16,8 +16,8 @@ Compared to the latest release [2.4.1](release_notes_241.html), the following bu ### Bugfixes This is a complete list of the -[merged pull requests](https://github.com/eclipse/ditto/pulls?q=is%3Apr+milestone%3A2.4.2), including the fixed bugs. +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A2.4.2), including the fixed bugs. -#### [Skip hostname verification check with self signed in kafka](https://github.com/eclipse/ditto/pull/1475) +#### [Skip hostname verification check with self signed in kafka](https://github.com/eclipse-ditto/ditto/pull/1475) Another required fix in order to connect to a Kafka with a self signed certificate. diff --git a/documentation/src/main/resources/pages/ditto/release_notes_300.md b/documentation/src/main/resources/pages/ditto/release_notes_300.md index ce620028691..62daecf59aa 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_300.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_300.md @@ -50,10 +50,10 @@ We want to especially highlight the following bugfixes also included: ### Changes -#### [New Ditto "thing" search index](https://github.com/eclipse/ditto/issues/1374) +#### [New Ditto "thing" search index](https://github.com/eclipse-ditto/ditto/issues/1374) Change reasoning documented in -[DADR-0008](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0008-wildcard-search-index.md). +[DADR-0008](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0008-wildcard-search-index.md). The Ditto 2.x implementation of the search index uses the [attribute pattern](https://www.mongodb.com/blog/post/building-with-patterns-the-attribute-pattern) for indexing data in MongoDB. @@ -74,10 +74,10 @@ query and update performance is improved and more stable than with the current a {% include note.html content="Be aware of the [migration note](#migration-building-up-new-search-index) before updating to Ditto 3.0." %} -#### [Removal of former "ditto-concierge" service](https://github.com/eclipse/ditto/issues/1339) +#### [Removal of former "ditto-concierge" service](https://github.com/eclipse-ditto/ditto/issues/1339) Change reasoning documented in -[DADR-0007](https://github.com/eclipse/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0007-concierge-removal.md). +[DADR-0007](https://github.com/eclipse-ditto/ditto/blob/master/documentation/src/main/resources/architecture/DADR-0007-concierge-removal.md). In order to reduce the complexity of message flows and the required networking "hops" in a Ditto cluster for a command to be authorized and applied, it was decided to eliminate the former "ditto-concierge" service. @@ -101,7 +101,7 @@ Ditto 3.0 requires less resources and provide lower latency + higher throughput #### Clear declaration and configuration of Ditto "Extension points" -As part of the ["ditto-concierge" service removal](https://github.com/eclipse/ditto/issues/1339) a new and common +As part of the ["ditto-concierge" service removal](https://github.com/eclipse-ditto/ditto/issues/1339) a new and common mechanism for [extending Ditto](installation-extending.html) was introduced. Examples of such extensions are: * adding additional, custom HTTP APIs to Ditto @@ -111,7 +111,7 @@ Examples of such extensions are: Check out the [documentation on extending Ditto](installation-extending.html) and the existing extension points (interfaces extending `DittoExtensionPoint`) to find out what can be extended and how to do it. -#### [Rewrite of Ditto managed MQTT connection to use reactive client](https://github.com/eclipse/ditto/pull/1411) +#### [Rewrite of Ditto managed MQTT connection to use reactive client](https://github.com/eclipse-ditto/ditto/pull/1411) Ditto's MQTT connection integration now consumes messages in a reactive manner. Together with throttling this effectively enables backpressure. @@ -119,10 +119,10 @@ Ditto's MQTT connection integration now consumes messages in a reactive manner. Improving Ditto's resilience is part of almost every release. Those improvements are included in 3.0: -* [Allow turning the akka SBR on/off during runtime](https://github.com/eclipse/ditto/pull/1373) -* [Implement graceful shutdown for http publisher actor](https://github.com/eclipse/ditto/pull/1381) +* [Allow turning the akka SBR on/off during runtime](https://github.com/eclipse-ditto/ditto/pull/1373) +* [Implement graceful shutdown for http publisher actor](https://github.com/eclipse-ditto/ditto/pull/1381) -#### [DevOps commands response consistency](https://github.com/eclipse/ditto/pull/1380) +#### [DevOps commands response consistency](https://github.com/eclipse-ditto/ditto/pull/1380) Devops commands error responses are fixed to have similar structure to non-error ones. Responses to requests with `"aggregate": false` are stripped to remove service name and instance (or layers of `"?"`) @@ -133,18 +133,18 @@ If devops commands were utilized, this will require a migration of the response {% include note.html content="Be aware of the [migration note](#migration-devops-commands-response-adjustments) before updating to Ditto 3.0." %} -#### [Make Ditto pubsub update operations simpler and more consistent](https://github.com/eclipse/ditto/pull/1427) +#### [Make Ditto pubsub update operations simpler and more consistent](https://github.com/eclipse-ditto/ditto/pull/1427) Simplify Ditto pubsub update operations to make sure that subscriptions are active before sending acknowledgements. ### New features -#### [Ability to search in JSON arrays and also in feature definitions](https://github.com/eclipse/ditto/pull/1396) +#### [Ability to search in JSON arrays and also in feature definitions](https://github.com/eclipse-ditto/ditto/pull/1396) [Searching in arrays](basic-search.html#search-queries-in-json-arrays) is now officially supported and enabled by default. -This also enables [support to search by feature definition](https://github.com/eclipse/ditto/pull/1417). +This also enables [support to search by feature definition](https://github.com/eclipse-ditto/ditto/pull/1417). You can use this e.g. in order to search for all things having a feature with a certain definition: ``` @@ -158,14 +158,14 @@ There were however still some open issues and bugs which lead to that the metada The following features, enhancements and fixes about [metadata](basic-metadata.html) are included in Ditto 3.0: -* [Retrieve thing metadata when not retrieving complete thing](https://github.com/eclipse/ditto/issues/772) -* [Metadata is not deleted when thing parts are deleted](https://github.com/eclipse/ditto/issues/829) -* [Make is possible to delete metadata from a thing](https://github.com/eclipse/ditto/issues/779) -* [Metadata cannot be set on sub-resources via HTTP](https://github.com/eclipse/ditto/issues/1146) -* [Add initial-metadata support to thing creation](https://github.com/eclipse/ditto/issues/884) -* [Search does not work for _metadata](https://github.com/eclipse/ditto/issues/1404) +* [Retrieve thing metadata when not retrieving complete thing](https://github.com/eclipse-ditto/ditto/issues/772) +* [Metadata is not deleted when thing parts are deleted](https://github.com/eclipse-ditto/ditto/issues/829) +* [Make is possible to delete metadata from a thing](https://github.com/eclipse-ditto/ditto/issues/779) +* [Metadata cannot be set on sub-resources via HTTP](https://github.com/eclipse-ditto/ditto/issues/1146) +* [Add initial-metadata support to thing creation](https://github.com/eclipse-ditto/ditto/issues/884) +* [Search does not work for _metadata](https://github.com/eclipse-ditto/ditto/issues/1404) -#### [New HTTP API for CRUD of connections](https://github.com/eclipse/ditto/issues/1406) +#### [New HTTP API for CRUD of connections](https://github.com/eclipse-ditto/ditto/issues/1406) With Ditto 2.x, connection management in Ditto could only be done via [piggyback devops commands](installation-operating.html#piggyback-commands). @@ -182,17 +182,17 @@ The latest live version of the UI can be found here: You can use it in order to e.g. connect to your Ditto installation to manage things, policies and even connections. Contributions: -* [Eclipse Ditto explorer UI](https://github.com/eclipse/ditto/pull/1397) -* [Ditto explorer UI: Improvements from review](https://github.com/eclipse/ditto/pull/1405) -* [Explorer UI - Add initial support for connections](https://github.com/eclipse/ditto/pull/1414) -* [added mechanism to build "ditto-ui" Docker image](https://github.com/eclipse/ditto/pull/1415) -* [Explorer UI - review improvements for connections](https://github.com/eclipse/ditto/pull/1418) -* [Explorer UI: add local_ditto_ide and ditto_sanbdox environments](https://github.com/eclipse/ditto/pull/1422) -* [Explorer UI - add support for policies](https://github.com/eclipse/ditto/pull/1430) -* [Explorer UI - Fix: Avoid storing credentials](https://github.com/eclipse/ditto/pull/1464) -* [Explorer UI - Improve message to feature and some WoT support](https://github.com/eclipse/ditto/pull/1455) - -#### [Support for EC signed JsonWebKeys (JWKs)](https://github.com/eclipse/ditto/pull/1432) +* [Eclipse Ditto explorer UI](https://github.com/eclipse-ditto/ditto/pull/1397) +* [Ditto explorer UI: Improvements from review](https://github.com/eclipse-ditto/ditto/pull/1405) +* [Explorer UI - Add initial support for connections](https://github.com/eclipse-ditto/ditto/pull/1414) +* [added mechanism to build "ditto-ui" Docker image](https://github.com/eclipse-ditto/ditto/pull/1415) +* [Explorer UI - review improvements for connections](https://github.com/eclipse-ditto/ditto/pull/1418) +* [Explorer UI: add local_ditto_ide and ditto_sanbdox environments](https://github.com/eclipse-ditto/ditto/pull/1422) +* [Explorer UI - add support for policies](https://github.com/eclipse-ditto/ditto/pull/1430) +* [Explorer UI - Fix: Avoid storing credentials](https://github.com/eclipse-ditto/ditto/pull/1464) +* [Explorer UI - Improve message to feature and some WoT support](https://github.com/eclipse-ditto/ditto/pull/1455) + +#### [Support for EC signed JsonWebKeys (JWKs)](https://github.com/eclipse-ditto/ditto/pull/1432) In Ditto 2.x the deserialization of an Elliptic Curve Json Web Token (JWT) failed, because Ditto assumed it to be an RSA token and missed the modulus and exponent information. @@ -204,32 +204,32 @@ With the [W3C Web of Things "Thing Description 1.1" standard](https://www.w3.org its final phases before official recommendation by the W3C, we think it is time to enable the [WoT Integration](basic-wot-integration.html) in Ditto by default and suggest it as the "default" type system for Ditto. -Together with some minor adjustments to the [Ditto WoT model](https://github.com/eclipse/ditto/tree/master/wot/model) to +Together with some minor adjustments to the [Ditto WoT model](https://github.com/eclipse-ditto/ditto/tree/master/wot/model) to the final changes to the 1.1 version of WoT TD, the following improvements also made it into Ditto 3.0: -* [Added WoT context extension ontologies in different formats](https://github.com/eclipse/ditto/pull/1442) -* [Apply WoT Ditto extension in skeleton and TD generation](https://github.com/eclipse/ditto/pull/1460) +* [Added WoT context extension ontologies in different formats](https://github.com/eclipse-ditto/ditto/pull/1442) +* [Apply WoT Ditto extension in skeleton and TD generation](https://github.com/eclipse-ditto/ditto/pull/1460) By using the Ditto WoT Extension Ontology located at [https://ditto.eclipseprojects.io/wot/ditto-extension](https://ditto.eclipseprojects.io/wot/ditto-extension), it is possible to define an additional `"category"` for WoT properties. That can for example be used to ease the migration from Vorto models, e.g. by defining a `"ditto:category": "configuration"` inside a WoT ThingModel. -#### [Make "default namespace" for creating new entities configurable](https://github.com/eclipse/ditto/pull/1372) +#### [Make "default namespace" for creating new entities configurable](https://github.com/eclipse-ditto/ditto/pull/1372) When e.g. creating new entities (things/policies) via `HTTP POST`, previously an empty namespace was used to create them in. -This namespace can now be [configured](https://github.com/eclipse/ditto/blob/master/edge/service/src/main/resources/ditto-edge-service.conf#L12-L13) +This namespace can now be [configured](https://github.com/eclipse-ditto/ditto/blob/master/edge/service/src/main/resources/ditto-edge-service.conf#L12-L13) via the environment variable `DITTO_DEFAULT_NAMESPACE` set to the "edge" services (ditto-gateway and ditto-connectivity). -#### [Provide custom namespace when creating things via HTTP POST](https://github.com/eclipse/ditto/issues/550) +#### [Provide custom namespace when creating things via HTTP POST](https://github.com/eclipse-ditto/ditto/issues/550) Provides the option to provide a custom namespace to create a new thing in when using `HTTP POST`. -#### [Make it possible to provide multiple OIDC issuer urls for a single configured openid-connect "prefix"](https://github.com/eclipse/ditto/pull/1465) +#### [Make it possible to provide multiple OIDC issuer urls for a single configured openid-connect "prefix"](https://github.com/eclipse-ditto/ditto/pull/1465) Configure multiple `issuer` endpoints for the same configured [openid-connect-provider](installation-operating.html#openid-connect). -#### [Addition of a "CloudEvents" mapper for mapping CE payloads in Ditto connections](https://github.com/eclipse/ditto/pull/1437) +#### [Addition of a "CloudEvents" mapper for mapping CE payloads in Ditto connections](https://github.com/eclipse-ditto/ditto/pull/1437) Adds a [CloudEvents](connectivity-mapping.html#cloudevents-mapper) in order to e.g. consume CloudEvents via Apache Kafka or MQTT and additionally to also publish CEs as well via all supported connectivity types. @@ -239,9 +239,9 @@ or MQTT and additionally to also publish CEs as well via all supported connectiv Several bugs in Ditto 2.4.x were fixed for 3.0.0. This is a complete list of the -* [merged pull requests for milestone 3.0.0](https://github.com/eclipse/ditto/pulls?q=is:pr+milestone:3.0.0) +* [merged pull requests for milestone 3.0.0](https://github.com/eclipse-ditto/ditto/pulls?q=is:pr+milestone:3.0.0) -Here as well for the Ditto Java Client: [merged pull requests for milestone 3.0.0](https://github.com/eclipse/ditto-clients/pulls?q=is:pr+milestone:3.0.0) +Here as well for the Ditto Java Client: [merged pull requests for milestone 3.0.0](https://github.com/eclipse-ditto/ditto-clients/pulls?q=is:pr+milestone:3.0.0) #### Passwords stored in the URI of connections to no longer need to be double encoded @@ -249,29 +249,29 @@ Previously for some passwords containing special characters like e.g. a `+` the before storing it to the [Connection](basic-connections.html) `URI`. This has been fixed by the PR: -* [Remove double decoding credentials of Connection URI passwords](https://github.com/eclipse/ditto/pull/1471) +* [Remove double decoding credentials of Connection URI passwords](https://github.com/eclipse-ditto/ditto/pull/1471) Also fixing the reported issue: -* [Basic auth in (mqtt) connection requires you to encode the username and password twice](https://github.com/eclipse/ditto/issues/1199) +* [Basic auth in (mqtt) connection requires you to encode the username and password twice](https://github.com/eclipse-ditto/ditto/issues/1199) {% include note.html content="Be aware of the [migration note](#migration-connection-uri-password-encoding) before updating to Ditto 3.0." %} -#### [When merging a feature, the normalized payload does not contain full feature](https://github.com/eclipse/ditto/issues/1446) +#### [When merging a feature, the normalized payload does not contain full feature](https://github.com/eclipse-ditto/ditto/issues/1446) When using the [Normalized mapper](connectivity-mapping.html#normalized-mapper) in Ditto connections together with [extra field enrichment](basic-enrichment.html), the outcome "merged" thing JSON structure could miss some information from the enriched "extra" data. This has been fixed. -#### [Fix that adding custom Java MessageMappers to Ditto via classpath is no longer possible](https://github.com/eclipse/ditto/issues/1463) +#### [Fix that adding custom Java MessageMappers to Ditto via classpath is no longer possible](https://github.com/eclipse-ditto/ditto/issues/1463) Writing own, Java based, `MessageMappers` and adding them to the classpath [as documented](connectivity-mapping.html#custom-java-based-implementation) did no longer work. This has been fixed and the documentation for doing exactly that has been updated. In addition, an example of how to provide a custom Protobuf payload mapper has been provided via -[custom-ditto-java-payload-mapper](https://github.com/eclipse/ditto-examples/tree/master/custom-ditto-java-payload-mapper). +[custom-ditto-java-payload-mapper](https://github.com/eclipse-ditto/ditto-examples/tree/master/custom-ditto-java-payload-mapper). ## Migration notes @@ -380,7 +380,7 @@ When [Restricting entity creation](installation-operating.html#restricting-entit the configuration for Ditto 3.0 has to be adjusted. As the configuration was part of the former "ditto-concierge" service (which was removed from Ditto 3.0), the configuration -is now located in [ditto-entity-creation.conf](https://github.com/eclipse/ditto/blob/master/internal/utils/config/src/main/resources/ditto-entity-creation.conf). +is now located in [ditto-entity-creation.conf](https://github.com/eclipse-ditto/ditto/blob/master/internal/utils/config/src/main/resources/ditto-entity-creation.conf). So the configuration keys changed * from: `ditto.concierge.enforcement.entity-creation` @@ -391,7 +391,7 @@ done for the "ditto-concierge" service. ### Migration: DevOps commands response adjustments -In [#1380](https://github.com/eclipse/ditto/pull/1380) the response of DevOps commands was consolidated to have a simpler +In [#1380](https://github.com/eclipse-ditto/ditto/pull/1380) the response of DevOps commands was consolidated to have a simpler response format. If DevOps command responses were used in prior Ditto versions, please adjust according to the [new response formats](installation-operating.html#devops-commands). @@ -415,8 +415,8 @@ double encoding **must be migrated** to use single encoding instead. Looking forward, the current plans for Ditto 3.1.0 are: -* [Support AMQP Message Annotations when extracting values for Headers](https://github.com/eclipse/ditto/issues/1390) -* [Policy imports](https://github.com/eclipse/ditto/issues/298) which will allow re-use of policies by importing existing ones +* [Support AMQP Message Annotations when extracting values for Headers](https://github.com/eclipse-ditto/ditto/issues/1390) +* [Policy imports](https://github.com/eclipse-ditto/ditto/issues/298) which will allow re-use of policies by importing existing ones * Perform a benchmark of Ditto 3.0 and provide a "tuning" chapter in the documentation as a reference to the commonly asked questions * how many Things Ditto can manage diff --git a/documentation/src/main/resources/slides/2018_02_07-virtualiot-meetup/index.html b/documentation/src/main/resources/slides/2018_02_07-virtualiot-meetup/index.html index 3879ba04d1b..8bdf308e188 100644 --- a/documentation/src/main/resources/slides/2018_02_07-virtualiot-meetup/index.html +++ b/documentation/src/main/resources/slides/2018_02_07-virtualiot-meetup/index.html @@ -594,7 +594,7 @@

thanks for attending

Bosch Software Innovations GmbH All rights reserved. | Imprint | Documentation | - GitHub | + GitHub | Sandbox
diff --git a/documentation/src/main/resources/slides/2018_05_23-meetup-iot-hessen/index.html b/documentation/src/main/resources/slides/2018_05_23-meetup-iot-hessen/index.html index aa2a389f315..8832e8b1e28 100644 --- a/documentation/src/main/resources/slides/2018_05_23-meetup-iot-hessen/index.html +++ b/documentation/src/main/resources/slides/2018_05_23-meetup-iot-hessen/index.html @@ -614,7 +614,7 @@

examples for orchestrations

Demo time!


-

GitHub sources

+

GitHub sources

@@ -683,7 +683,7 @@

Q & A

Bosch Software Innovations GmbH All rights reserved. | Imprint | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2018_10_22-eclipse-iot-wg-f2f-ludwigsburg/index.html b/documentation/src/main/resources/slides/2018_10_22-eclipse-iot-wg-f2f-ludwigsburg/index.html index 3711b9dc217..b9cd0e98936 100644 --- a/documentation/src/main/resources/slides/2018_10_22-eclipse-iot-wg-f2f-ludwigsburg/index.html +++ b/documentation/src/main/resources/slides/2018_10_22-eclipse-iot-wg-f2f-ludwigsburg/index.html @@ -248,7 +248,7 @@

Stats

Questions via GitHub - 15 + 15 (increasing interest) @@ -287,7 +287,7 @@

Collaboration

  • Eclipse Hono: ✔
    (telemetry/events/command&control)
  • Eclipse Mosquitto: ✔
    (connection to MQTT 3.1.1 brokers)
  • Eclipse Vorto:
    on our agenda to integrate in order to - validate structure of digital twins: #107, #247
  • + validate structure of digital twins: #107, #247 @@ -300,7 +300,7 @@

    Collaboration

    Bosch Software Innovations GmbH All rights reserved. | Imprint | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2019_10_21-eclipse-iot-wg-f2f-ludwigsburg/index.html b/documentation/src/main/resources/slides/2019_10_21-eclipse-iot-wg-f2f-ludwigsburg/index.html index 33709e6a956..9c31b557605 100644 --- a/documentation/src/main/resources/slides/2019_10_21-eclipse-iot-wg-f2f-ludwigsburg/index.html +++ b/documentation/src/main/resources/slides/2019_10_21-eclipse-iot-wg-f2f-ludwigsburg/index.html @@ -307,7 +307,7 @@

    Challenges

    Bosch Software Innovations GmbH All rights reserved. | Imprint | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2020_04_16-ttn-virtual-conference/index.html b/documentation/src/main/resources/slides/2020_04_16-ttn-virtual-conference/index.html index fee2fe1c3eb..95fb0ced1a3 100644 --- a/documentation/src/main/resources/slides/2020_04_16-ttn-virtual-conference/index.html +++ b/documentation/src/main/resources/slides/2020_04_16-ttn-virtual-conference/index.html @@ -589,7 +589,7 @@

    thanks to The Things Network for organising this virtual conference

    Links:

    @@ -603,7 +603,7 @@

    thanks to The Things Network for organising this virtual conference

    Copyright ©2020 Bosch.IO GmbH All rights reserved. | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2020_07_28-iot-wg-status-update/index.html b/documentation/src/main/resources/slides/2020_07_28-iot-wg-status-update/index.html index 4d62558df58..ed3bfe740d8 100644 --- a/documentation/src/main/resources/slides/2020_07_28-iot-wg-status-update/index.html +++ b/documentation/src/main/resources/slides/2020_07_28-iot-wg-status-update/index.html @@ -313,7 +313,7 @@

    Roadmap

    Copyright ©2020 Bosch.IO GmbH All rights reserved. | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2020_10_19-eclipse-iot-wg-community-day/index.html b/documentation/src/main/resources/slides/2020_10_19-eclipse-iot-wg-community-day/index.html index 7c9fb9178cb..f9f92594b70 100644 --- a/documentation/src/main/resources/slides/2020_10_19-eclipse-iot-wg-community-day/index.html +++ b/documentation/src/main/resources/slides/2020_10_19-eclipse-iot-wg-community-day/index.html @@ -311,7 +311,7 @@

    Achievements

  • - Initial contribution of a Ditto Golang client + Initial contribution of a Ditto Golang client
  • LoRaWAN Virtual Conference: @@ -337,7 +337,7 @@

    Obstacles

    Roadmap

      -
    • 1.4.0 +
    • 1.4.0 (12/2020):
      • enhance twin by "desired" properties
      • @@ -362,7 +362,7 @@

        Roadmap

        Copyright ©2020 Bosch.IO GmbH All rights reserved. | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2021_05_ditto-introduction-deck/index.html b/documentation/src/main/resources/slides/2021_05_ditto-introduction-deck/index.html index 6b7f4030493..eb064691626 100644 --- a/documentation/src/main/resources/slides/2021_05_ditto-introduction-deck/index.html +++ b/documentation/src/main/resources/slides/2021_05_ditto-introduction-deck/index.html @@ -579,7 +579,7 @@

        Wrap up

        Links:

        @@ -593,7 +593,7 @@

        Wrap up

        Copyright ©2021 Bosch.IO GmbH All rights reserved. | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2021_06_ditto-20-overview/index.html b/documentation/src/main/resources/slides/2021_06_ditto-20-overview/index.html index ec926345d90..3d16be605f8 100644 --- a/documentation/src/main/resources/slides/2021_06_ditto-20-overview/index.html +++ b/documentation/src/main/resources/slides/2021_06_ditto-20-overview/index.html @@ -287,7 +287,7 @@

        Looking ahead

        Links:

        @@ -301,7 +301,7 @@

        Looking ahead

        Copyright ©2021 Bosch.IO GmbH All rights reserved. | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2021_06_ditto-in-20-min/index.html b/documentation/src/main/resources/slides/2021_06_ditto-in-20-min/index.html index c43230282e1..08fca90e49f 100644 --- a/documentation/src/main/resources/slides/2021_06_ditto-in-20-min/index.html +++ b/documentation/src/main/resources/slides/2021_06_ditto-in-20-min/index.html @@ -414,7 +414,7 @@

        Wrap up

        Links:

        @@ -428,7 +428,7 @@

        Wrap up

        Copyright ©2021 Bosch.IO GmbH All rights reserved. | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2021_10_25-eclipse-iot-wg-community-day/index.html b/documentation/src/main/resources/slides/2021_10_25-eclipse-iot-wg-community-day/index.html index 0a03b651211..d17d5882b5e 100644 --- a/documentation/src/main/resources/slides/2021_10_25-eclipse-iot-wg-community-day/index.html +++ b/documentation/src/main/resources/slides/2021_10_25-eclipse-iot-wg-community-day/index.html @@ -287,13 +287,13 @@

        Current topics

      • W3C WoT (Web of Things) integration:
      • - Initial contribution of a Ditto Python client + Initial contribution of a Ditto Python client
    @@ -302,7 +302,7 @@

    Current topics

    Roadmap

      -
    • 2.2.0 (late 2021): +
    • 2.2.0 (late 2021):
      • filter for lifecycle events (e.g. twin created/deleted)
      • HTTP API for “live” commands
      • @@ -325,7 +325,7 @@

        Roadmap

        Copyright ©2021 Bosch.IO GmbH All rights reserved. | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2022_10_24_eclipse-iot-wg-community-day/index.html b/documentation/src/main/resources/slides/2022_10_24_eclipse-iot-wg-community-day/index.html index afc7f8cb0a4..feeb3218875 100644 --- a/documentation/src/main/resources/slides/2022_10_24_eclipse-iot-wg-community-day/index.html +++ b/documentation/src/main/resources/slides/2022_10_24_eclipse-iot-wg-community-day/index.html @@ -322,7 +322,7 @@

        Current topics

        Roadmap

        @@ -536,7 +536,7 @@

        Links

        Copyright ©2022 Bosch.IO GmbH All rights reserved. | Documentation | - GitHub | + GitHub | Sandbox diff --git a/documentation/src/main/resources/slides/2023_01_ditto-in-30-min/index.html b/documentation/src/main/resources/slides/2023_01_ditto-in-30-min/index.html index 825e4784080..115339d5c21 100644 --- a/documentation/src/main/resources/slides/2023_01_ditto-in-30-min/index.html +++ b/documentation/src/main/resources/slides/2023_01_ditto-in-30-min/index.html @@ -497,7 +497,7 @@

        Links

    @@ -510,7 +510,7 @@

    Links

    Copyright ©2023 Bosch.IO GmbH All rights reserved. | Documentation | - GitHub | + GitHub | Sandbox diff --git a/legal/NOTICE.md b/legal/NOTICE.md index 64523368e2d..c5b2026dc69 100644 --- a/legal/NOTICE.md +++ b/legal/NOTICE.md @@ -24,7 +24,7 @@ SPDX-License-Identifier: EPL-2.0 # Source Code -* https://github.com/eclipse/ditto +* https://github.com/eclipse-ditto/ditto # Third-party Content diff --git a/pom.xml b/pom.xml index b8c9190c220..f5998d5a886 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ GitHub Issues - https://github.com/eclipse/ditto/issues + https://github.com/eclipse-ditto/ditto/issues From 63efb2b7d5ca6d8de1a7ced7cc4fccd7655b92fe Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Mon, 30 Jan 2023 09:16:54 +0100 Subject: [PATCH 030/173] added streaming protocol usage example to history documentation Signed-off-by: Thomas Jaeckle --- .run/ConnectivityService.run.xml | 22 +---- .run/Ditto.run.xml | 12 --- .run/GatewayService.run.xml | 18 ---- .run/PoliciesService.run.xml | 23 +---- .run/SearchService.run.xml | 18 ---- .run/ThingsService.run.xml | 21 +---- .../resources/pages/ditto/basic-history.md | 89 +++++++++++++++++-- 7 files changed, 91 insertions(+), 112 deletions(-) diff --git a/.run/ConnectivityService.run.xml b/.run/ConnectivityService.run.xml index fd0fa2ff98d..19697e5c888 100644 --- a/.run/ConnectivityService.run.xml +++ b/.run/ConnectivityService.run.xml @@ -1,31 +1,15 @@ - + + - + \ No newline at end of file diff --git a/.run/Ditto.run.xml b/.run/Ditto.run.xml index ac5bcbb86b2..6aacf9255db 100644 --- a/.run/Ditto.run.xml +++ b/.run/Ditto.run.xml @@ -1,15 +1,3 @@ - diff --git a/.run/GatewayService.run.xml b/.run/GatewayService.run.xml index 7884b94f5bc..de83356697b 100644 --- a/.run/GatewayService.run.xml +++ b/.run/GatewayService.run.xml @@ -1,15 +1,3 @@ - @@ -18,12 +6,6 @@ \ No newline at end of file diff --git a/.run/SearchService.run.xml b/.run/SearchService.run.xml index 98ee0fb7b41..25444065eaa 100644 --- a/.run/SearchService.run.xml +++ b/.run/SearchService.run.xml @@ -1,15 +1,3 @@ - @@ -23,12 +11,6 @@ \ No newline at end of file diff --git a/documentation/src/main/resources/pages/ditto/basic-history.md b/documentation/src/main/resources/pages/ditto/basic-history.md index 89c9cec0e9b..fdf9c908c7c 100644 --- a/documentation/src/main/resources/pages/ditto/basic-history.md +++ b/documentation/src/main/resources/pages/ditto/basic-history.md @@ -12,11 +12,11 @@ Starting with **Eclipse Ditto 3.2.0**, APIs for retrieving the history of the fo The capabilities of these APIs are the following: -| Entity | Retrieving entity at a specific revision or timestamp | Streaming modification events of an entity specifying from/to revision/timestamp | -|------------|-------------------------------------------------------|----------------------------------------------------------------------------------| -| Thing | ✓ | ✓ | -| Policy | ✓ | ✓ | -| Connection | ✓ | no | +| Entity | [Retrieving entity at a specific revision or timestamp](#retrieving-entity-from-history) | [Streaming modification events of an entity specifying from/to revision/timestamp](#streaming-historical-events-of-entity) | +|------------|------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------| +| Thing | ✓ | ✓ | +| Policy | ✓ | ✓ | +| Connection | ✓ | no | {% include note.html content="Ditto's history API capabilities are not comparable with the features of a time series database. E.g. no aggregations on or compactions of the historical data can be done." %} @@ -53,7 +53,7 @@ curl -u ditto:ditto 'http://localhost:8080/api/2/policies/org.eclipse.ditto:poli --header 'at-historical-revision: 1' # Access a connection: -curl -u ditto:ditto 'http://localhost:8080/api/2/connections/some-connection-1' \ +curl -u devops:foobar 'http://localhost:8080/api/2/connections/some-connection-1' \ --header 'at-historical-revision: 1' ``` @@ -78,7 +78,7 @@ curl -u ditto:ditto 'http://localhost:8080/api/2/policies/org.eclipse.ditto:poli --header 'at-historical-timestamp: 2022-10-24T06:11:15Z' # Access a connection: -curl -u ditto:ditto 'http://localhost:8080/api/2/connections/some-connection-1' \ +curl -u devops:foobar 'http://localhost:8080/api/2/connections/some-connection-1' \ --header 'at-historical-timestamp: 2022-10-24T07:11Z' ``` @@ -144,6 +144,81 @@ Please inspect the [protocol specification of DittoProtocol messages for streami to find out how to stream historical (persisted) events via DittoProtocol. Using the DittoProtocol, historical events can be streamed either via WebSocket or connections. +Example protocol interaction for retrieving the persisted events of a thing: + +**First:** Subscribe for the persisted events of a thing +```json +{ + "topic": "org.eclipse.ditto/thing-2/things/twin/streaming/subscribeForPersistedEvents", + "path": "/", + "headers": {}, + "value": { + "fromHistoricalRevision": 1, + "toHistoricalRevision": 10 + } +} +``` + +Alternatively to `fromHistoricalRevision` and `toHistoricalRevision`, also a timestamp based range may be used: +`fromHistoricalTimestamp` and `toHistoricalTimestamp`. +The "to" can be omitted in order to receive all events up to the current revision or timestamp. + +As a result, the following `created` event is received as response: +```json +{ + "topic": "org.eclipse.ditto/thing-2/things/twin/streaming/created", + "path": "/", + "headers": {}, + "value": { + "subscriptionId": "0" + } +} +``` + +**Second:** Once the streaming subscription is confirmed to be created, request demand (of how many events to get streamed), +referencing the `subscriptionId`: +```json +{ + "topic": "org.eclipse.ditto/thing-2/things/twin/streaming/request", + "path": "/", + "headers": {}, + "value": { + "subscriptionId": "0", + "demand": 25 + } +} +``` + +The backend will start sending the requested persisted events as `next` messages: +```json +{ + "topic": "org.eclipse.ditto/thing-2/things/twin/streaming/next", + "path": "/", + "headers": {}, + "value": { + "subscriptionId": "0", + "item": { + + } + } +} +``` + +It will do so either until all existing events were sent, in that case a `complete` event is sent: +```json +{ + "topic": "org.eclipse.ditto/thing-2/things/twin/streaming/complete", + "path": "/", + "headers": {}, + "value": { + "subscriptionId": "0" + } +} +``` + +Or it will stop after the `demand` was fulfilled, waiting for the requester to claim more demand with a new `request` +message. + ## Configuring historical headers to persist From 889b14b404f89d3bd79e36a006c00a8177ea98f4 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Tue, 7 Feb 2023 14:48:30 +0100 Subject: [PATCH 031/173] increased defaults for configured history-retention-duration Signed-off-by: Thomas Jaeckle --- connectivity/service/src/main/resources/connectivity.conf | 2 +- .../src/main/resources/pages/ditto/installation-operating.md | 2 +- policies/service/src/main/resources/policies.conf | 2 +- things/service/src/main/resources/things.conf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/connectivity/service/src/main/resources/connectivity.conf b/connectivity/service/src/main/resources/connectivity.conf index 353972cdc5f..e2bc8a317a1 100644 --- a/connectivity/service/src/main/resources/connectivity.conf +++ b/connectivity/service/src/main/resources/connectivity.conf @@ -701,7 +701,7 @@ ditto { # allowed to remove them in scope of cleanup. # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read # journal. - history-retention-duration = 0d + history-retention-duration = 30d history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} # quiet-period defines how long to stay in a state where the background cleanup is not yet started diff --git a/documentation/src/main/resources/pages/ditto/installation-operating.md b/documentation/src/main/resources/pages/ditto/installation-operating.md index ef0605abb5b..fb992b7ce28 100644 --- a/documentation/src/main/resources/pages/ditto/installation-operating.md +++ b/documentation/src/main/resources/pages/ditto/installation-operating.md @@ -812,7 +812,7 @@ cleanup { # allowed to remove them in scope of cleanup. # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read # journal. - history-retention-duration = 0d + history-retention-duration = 3d history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} # quiet-period defines how long to stay in a state where the background cleanup is not yet started diff --git a/policies/service/src/main/resources/policies.conf b/policies/service/src/main/resources/policies.conf index 0f1a79e8065..ea441dfee6d 100755 --- a/policies/service/src/main/resources/policies.conf +++ b/policies/service/src/main/resources/policies.conf @@ -121,7 +121,7 @@ ditto { # allowed to remove them in scope of cleanup. # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read # journal. - history-retention-duration = 0d + history-retention-duration = 30d history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} # quiet-period defines how long to stay in a state where the background cleanup is not yet started diff --git a/things/service/src/main/resources/things.conf b/things/service/src/main/resources/things.conf index 0e3e3237e74..0b3d818f8ca 100755 --- a/things/service/src/main/resources/things.conf +++ b/things/service/src/main/resources/things.conf @@ -86,7 +86,7 @@ ditto { # allowed to remove them in scope of cleanup. # If this e.g. is set to 30d - then effectively an event history of 30 days would be available via the read # journal. - history-retention-duration = 0d + history-retention-duration = 3d history-retention-duration = ${?CLEANUP_HISTORY_RETENTION_DURATION} # quiet-period defines how long to stay in a state where the background cleanup is not yet started From d53f630bf5e1b1bd92fee26dab8deb6bc6ded3b7 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Tue, 7 Feb 2023 16:53:01 +0100 Subject: [PATCH 032/173] provide release notes for Ditto bugfix release 3.1.2 Signed-off-by: Thomas Jaeckle --- .../pages/ditto/release_notes_311.md | 4 +-- .../pages/ditto/release_notes_312.md | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 documentation/src/main/resources/pages/ditto/release_notes_312.md diff --git a/documentation/src/main/resources/pages/ditto/release_notes_311.md b/documentation/src/main/resources/pages/ditto/release_notes_311.md index 83d3759a627..22bb47f3861 100644 --- a/documentation/src/main/resources/pages/ditto/release_notes_311.md +++ b/documentation/src/main/resources/pages/ditto/release_notes_311.md @@ -7,11 +7,11 @@ summary: "Version 3.1.1 of Eclipse Ditto, released on 05.01.2023" permalink: release_notes_311.html --- -This is a bugfix release, no new features since [3.0.0](release_notes_300.html) were added. +This is a bugfix release, no new features since [3.1.0](release_notes_310.html) were added. ## Changelog -Compared to the latest release [3.0.0](release_notes_300.html), the following bugfixes were added. +Compared to the latest release [3.1.0](release_notes_310.html), the following bugfixes were added. ### Bugfixes diff --git a/documentation/src/main/resources/pages/ditto/release_notes_312.md b/documentation/src/main/resources/pages/ditto/release_notes_312.md new file mode 100644 index 00000000000..831404f639e --- /dev/null +++ b/documentation/src/main/resources/pages/ditto/release_notes_312.md @@ -0,0 +1,31 @@ +--- +title: Release notes 3.1.2 +tags: [release_notes] +published: true +keywords: release notes, announcements, changelog +summary: "Version 3.1.2 of Eclipse Ditto, released on 08.02.2023" +permalink: release_notes_312.html +--- + +This is a bugfix release, no new features since [3.1.1](release_notes_311.html) were added. + +## Changelog + +Compared to the latest release [3.1.1](release_notes_311.html), the following bugfixes were added. + +### Bugfixes + +This is a complete list of the +[merged pull requests](https://github.com/eclipse-ditto/ditto/pulls?q=is%3Apr+milestone%3A3.1.2). + +#### [Outgoing mqtt connections fail to publish messages when tracing is enabled](https://github.com/eclipse-ditto/ditto/issues/1563) + +When [tracing](installation-operating.html#tracing) was enabled for a Ditto installation, outbound MQTT messages could +not be published due to an exception caused by an empty user property. +This was fixed. + +#### [Fixed that a missing (deleted) referenced policy of a policy import caused logging ERRORs](https://github.com/eclipse-ditto/ditto/pull/1571) + +When a policy referenced in a [policy import](basic-policy.html#policy-imports) was deleted, this caused logging `ERROR`s +in the [search](architecture-services-things-search.html) service. +This was fixed. From a00dd2e91a5c73cfe58a8b731f51ae0dde758e98 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Wed, 8 Feb 2023 09:28:43 +0100 Subject: [PATCH 033/173] updated sidebar to inlcude release 3.1.2 Signed-off-by: Thomas Jaeckle --- .../src/main/resources/_data/sidebars/ditto_sidebar.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml b/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml index e114dd11074..689dadad93b 100644 --- a/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml +++ b/documentation/src/main/resources/_data/sidebars/ditto_sidebar.yml @@ -23,6 +23,9 @@ entries: - title: Release Notes output: web folderitems: + - title: 3.1.2 + url: /release_notes_312.html + output: web - title: 3.1.1 url: /release_notes_311.html output: web From cc8125ba73b1dd4bd3bf560832cc04cb882da37d Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Wed, 8 Feb 2023 11:11:24 +0100 Subject: [PATCH 034/173] updated default ditto.protocol.blocklist to exclude headers "accept-encoding" and "x-forwarded-scheme" Signed-off-by: Thomas Jaeckle --- internal/utils/config/src/main/resources/ditto-protocol.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/utils/config/src/main/resources/ditto-protocol.conf b/internal/utils/config/src/main/resources/ditto-protocol.conf index 75fc78f98d8..a7714e1dd4b 100644 --- a/internal/utils/config/src/main/resources/ditto-protocol.conf +++ b/internal/utils/config/src/main/resources/ditto-protocol.conf @@ -5,5 +5,7 @@ ditto.protocol { "cache-control", "connection", "timeout-access", + "accept-encoding", + "x-forwarded-scheme", ] } From d4f3c7cf454a47285bec6e72e8d04900a3830553 Mon Sep 17 00:00:00 2001 From: Andrey Balarev Date: Thu, 9 Feb 2023 14:41:15 +0200 Subject: [PATCH 035/173] Hono connection address suffix added - hono tenant id Signed-off-by: Andrey Balarev --- .../hono/DefaultHonoConnectionFactory.java | 15 ++++++++-- .../DefaultHonoConnectionFactoryTest.java | 28 ++++++++++--------- .../ConnectionPersistenceActorTest.java | 21 ++++++++++---- .../hono-connection-custom-expected.json | 14 +++++----- .../connectivity-protocol-bindings-hono.md | 20 ++++++++----- 5 files changed, 63 insertions(+), 35 deletions(-) diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactory.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactory.java index 18c806b30fc..99043999f82 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactory.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactory.java @@ -16,6 +16,8 @@ import java.text.MessageFormat; import java.util.Set; +import org.eclipse.ditto.connectivity.model.Connection; +import org.eclipse.ditto.connectivity.model.ConnectionId; import org.eclipse.ditto.connectivity.model.HonoAddressAlias; import org.eclipse.ditto.connectivity.model.UserPasswordCredentials; import org.eclipse.ditto.connectivity.service.config.DefaultHonoConfig; @@ -33,6 +35,8 @@ public final class DefaultHonoConnectionFactory extends HonoConnectionFactory { private final HonoConfig honoConfig; + private ConnectionId connectionId; + /** * Constructs a {@code DefaultHonoConnectionFactory} for the specified arguments. * @@ -44,6 +48,11 @@ public DefaultHonoConnectionFactory(final ActorSystem actorSystem, final Config honoConfig = new DefaultHonoConfig(actorSystem); } + @Override + protected void preConversion(final Connection honoConnection) { + connectionId = honoConnection.getId(); + } + @Override public URI getBaseUri() { return honoConfig.getBaseUri(); @@ -76,12 +85,14 @@ protected UserPasswordCredentials getCredentials() { @Override protected String resolveSourceAddress(final HonoAddressAlias honoAddressAlias) { - return MessageFormat.format("hono.{0}", honoAddressAlias.getAliasValue()); + return MessageFormat.format("hono.{0}.{1}", + honoAddressAlias.getAliasValue(), connectionId); } @Override protected String resolveTargetAddress(final HonoAddressAlias honoAddressAlias) { - return MessageFormat.format("hono.{0}/'{{thing:id}}'", honoAddressAlias.getAliasValue()); + return MessageFormat.format("hono.{0}.{1}/'{{thing:id}}'", + honoAddressAlias.getAliasValue(), connectionId); } } diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java index 86a6bd2443e..d2b7344b440 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/hono/DefaultHonoConnectionFactoryTest.java @@ -30,6 +30,7 @@ import org.assertj.core.api.Assertions; import org.eclipse.ditto.connectivity.model.Connection; +import org.eclipse.ditto.connectivity.model.ConnectionId; import org.eclipse.ditto.connectivity.model.ConnectivityModelFactory; import org.eclipse.ditto.connectivity.model.HonoAddressAlias; import org.eclipse.ditto.connectivity.model.ReplyTarget; @@ -120,6 +121,7 @@ private Connection getExpectedHonoConnection(final Connection originalConnection "correlation-id", "{{ header:correlation-id }}", "subject", "{{ header:subject | fn:default(topic:action-subject) }}" ); + final var connectionId = originalConnection.getId(); return ConnectivityModelFactory.newConnectionBuilder(originalConnection) .uri(honoConfig.getBaseUri().toString().replaceFirst("(\\S+://)(\\S+)", "$1" + URLEncoder.encode(honoConfig.getUserPasswordCredentials().getUsername(), StandardCharsets.UTF_8) @@ -133,22 +135,22 @@ private Connection getExpectedHonoConnection(final Connection originalConnection ) .setSources(List.of( ConnectivityModelFactory.newSourceBuilder(sourcesByAddress.get(TELEMETRY.getAliasValue())) - .addresses(Set.of(getExpectedResolvedSourceAddress(TELEMETRY))) + .addresses(Set.of(getExpectedResolvedSourceAddress(TELEMETRY, connectionId))) .replyTarget(ReplyTarget.newBuilder() - .address(getExpectedResolvedCommandTargetAddress()) + .address(getExpectedResolvedCommandTargetAddress(connectionId)) .headerMapping(commandReplyTargetHeaderMapping) .build()) .build(), ConnectivityModelFactory.newSourceBuilder(sourcesByAddress.get(EVENT.getAliasValue())) - .addresses(Set.of(getExpectedResolvedSourceAddress(EVENT))) + .addresses(Set.of(getExpectedResolvedSourceAddress(EVENT, connectionId))) .replyTarget(ReplyTarget.newBuilder() - .address(getExpectedResolvedCommandTargetAddress()) + .address(getExpectedResolvedCommandTargetAddress(connectionId)) .headerMapping(commandReplyTargetHeaderMapping) .build()) .build(), ConnectivityModelFactory.newSourceBuilder( sourcesByAddress.get(COMMAND_RESPONSE.getAliasValue())) - .addresses(Set.of(getExpectedResolvedSourceAddress(COMMAND_RESPONSE))) + .addresses(Set.of(getExpectedResolvedSourceAddress(COMMAND_RESPONSE, connectionId))) .headerMapping(ConnectivityModelFactory.newHeaderMapping(Map.of( "correlation-id", "{{ header:correlation-id }}", "status", "{{ header:status }}" @@ -157,8 +159,8 @@ private Connection getExpectedHonoConnection(final Connection originalConnection )) .setTargets(List.of( ConnectivityModelFactory.newTargetBuilder(targets.get(0)) - .address(getExpectedResolvedCommandTargetAddress()) - .originalAddress(getExpectedResolvedCommandTargetAddress()) + .address(getExpectedResolvedCommandTargetAddress(connectionId)) + .originalAddress(getExpectedResolvedCommandTargetAddress(connectionId)) .headerMapping(ConnectivityModelFactory.newHeaderMapping( Stream.concat( basicAdditionalTargetHeaderMappingEntries.entrySet().stream(), @@ -168,8 +170,8 @@ private Connection getExpectedHonoConnection(final Connection originalConnection )) .build(), ConnectivityModelFactory.newTargetBuilder(targets.get(1)) - .address(getExpectedResolvedCommandTargetAddress()) - .originalAddress(getExpectedResolvedCommandTargetAddress()) + .address(getExpectedResolvedCommandTargetAddress(connectionId)) + .originalAddress(getExpectedResolvedCommandTargetAddress(connectionId)) .headerMapping(ConnectivityModelFactory.newHeaderMapping( basicAdditionalTargetHeaderMappingEntries )) @@ -184,12 +186,12 @@ private static Map getSourcesByAddress(final Iterable so return result; } - private static String getExpectedResolvedSourceAddress(final HonoAddressAlias honoAddressAlias) { - return "hono." + honoAddressAlias.getAliasValue(); + private static String getExpectedResolvedSourceAddress(final HonoAddressAlias honoAddressAlias, final ConnectionId connectionId) { + return "hono." + honoAddressAlias.getAliasValue() + "." + connectionId; } - private static String getExpectedResolvedCommandTargetAddress() { - return "hono." + HonoAddressAlias.COMMAND.getAliasValue() + "/{{thing:id}}"; + private static String getExpectedResolvedCommandTargetAddress(final ConnectionId connectionId) { + return "hono." + HonoAddressAlias.COMMAND.getAliasValue() + "." + connectionId + "/{{thing:id}}"; } } diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java index 31d1320b877..29e65232d02 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java @@ -27,6 +27,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; import org.eclipse.ditto.base.api.persistence.cleanup.CleanupPersistence; @@ -88,6 +90,7 @@ import org.eclipse.ditto.internal.utils.test.Retry; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.thingsearch.model.signals.commands.subscription.CreateSubscription; import org.junit.Before; import org.junit.ClassRule; @@ -213,11 +216,11 @@ public void testConnection() { @Test public void testConnectionTypeHono() throws IOException { //GIVEN - final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json") + final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json", null) .toBuilder() .id(connectionId) .build(); - final var expectedHonoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-expected.json") + final var expectedHonoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-expected.json", connectionId) .toBuilder() .id(connectionId) .build(); @@ -243,7 +246,7 @@ public void testConnectionTypeHono() throws IOException { @Test public void testRestartByConnectionType() throws IOException { // GIVEN - final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json"); + final var honoConnection = generateConnectionObjectFromJsonFile("hono-connection-custom-test.json", null); mockClientActorProbe.setAutoPilot(new TestActor.AutoPilot() { @Override public TestActor.AutoPilot run(final ActorRef sender, final Object msg) { @@ -283,13 +286,19 @@ public TestActor.AutoPilot run(final ActorRef sender, final Object msg) { }); } - private static Connection generateConnectionObjectFromJsonFile(final String fileName) throws IOException { + private static Connection generateConnectionObjectFromJsonFile(final String fileName, + @Nullable ConnectionId connectionId) throws IOException { final var testClassLoader = DefaultHonoConnectionFactoryTest.class.getClassLoader(); try (final var connectionJsonFileStreamReader = new InputStreamReader( testClassLoader.getResourceAsStream(fileName) )) { - return ConnectivityModelFactory.connectionFromJson( - JsonFactory.readFrom(connectionJsonFileStreamReader).asObject()); + JsonObject jsonObject = JsonFactory.readFrom(connectionJsonFileStreamReader).asObject(); + var connId = jsonObject.getValue("id"); + if (connectionId != null && connId.isPresent()) { + var jsonString = jsonObject.formatAsString().replace(connId.get().asString(), connectionId); + jsonObject = JsonFactory.readFrom(jsonString).asObject(); + } + return ConnectivityModelFactory.connectionFromJson(jsonObject); } } diff --git a/connectivity/service/src/test/resources/hono-connection-custom-expected.json b/connectivity/service/src/test/resources/hono-connection-custom-expected.json index 649861792bb..6ccab3c12a8 100644 --- a/connectivity/service/src/test/resources/hono-connection-custom-expected.json +++ b/connectivity/service/src/test/resources/hono-connection-custom-expected.json @@ -7,7 +7,7 @@ "sources": [ { "addresses": [ - "hono.telemetry" + "hono.telemetry.test-connection-id" ], "consumerCount": 1, "qos": 0, @@ -32,7 +32,7 @@ "implicitStandaloneThingCreation" ], "replyTarget": { - "address": "hono.command/{{thing:id}}", + "address": "hono.command.test-connection-id/{{thing:id}}", "headerMapping": { "device_id": "custom_value1", "user_key1": "user_value1", @@ -48,7 +48,7 @@ }, { "addresses": [ - "hono.event" + "hono.event.test-connection-id" ], "consumerCount": 1, "qos": 1, @@ -72,7 +72,7 @@ "implicitStandaloneThingCreation" ], "replyTarget": { - "address": "hono.command/{{thing:id}}", + "address": "hono.command.test-connection-id/{{thing:id}}", "headerMapping": { "device_id": "{{ thing:id }}", "subject": "custom_value2", @@ -88,7 +88,7 @@ }, { "addresses": [ - "hono.command_response" + "hono.command_response.test-connection-id" ], "consumerCount": 1, "qos": 0, @@ -120,7 +120,7 @@ ], "targets": [ { - "address": "hono.command/{{thing:id}}", + "address": "hono.command.test-connection-id/{{thing:id}}", "topics": [ "_/_/things/live/messages", "_/_/things/live/commands" @@ -137,7 +137,7 @@ } }, { - "address": "hono.command/{{thing:id}}", + "address": "hono.command.test-connection-id/{{thing:id}}", "topics": [ "_/_/things/twin/events", "_/_/things/live/events" diff --git a/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-hono.md b/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-hono.md index c6fd325131b..940d6a8c0be 100644 --- a/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-hono.md +++ b/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-hono.md @@ -8,13 +8,16 @@ permalink: connectivity-protocol-bindings-hono.html Consume messages from Eclipse Hono through Apache Kafka brokers and send messages to Eclipse Hono the same manner as [Kafka connection](connectivity-protocol-bindings-kafka2.html) does. -This connection type is created just for convenience - to avoid the need the user to be aware of the specific +This connection type is implemented just for convenience - to avoid the need the user to be aware of the specific header mappings, address formats and Kafka specificConfig, which are required to connect to Eclipse Hono. These specifics are applied automatically at runtime for the connections of type Hono. Hono connection is based on Kafka connection and uses it behind the scenes, so most of the [Kafka connection documentation](connectivity-protocol-bindings-kafka2.html) is valid for Hono connection too, -but with the following specifics (exceptions): +but with some exceptions, described bellow. + +#### Important note +During the creation of hono connection, the connection ID must be provided to be the same as `Hono-tenantId`. This is needed to match the Kafka topics (aka connection addresses) to the topics on which Hono will send and listen to. See bellow sections [Source addresses](#source-addresses), [Source reply target](#source-reply-target) and [Target Address](#target-address) ## Specific Hono connection configuration @@ -30,20 +33,23 @@ protocol identifier and the host name of `base-uri` to form the connection URI l Note: If any of these parameters has to be changed, the service must be restarted to apply the new values. - ### Source format #### Source addresses For a Hono connection source "addresses" are specified as aliases, which are resolved at runtime to Kafka topics to subscribe to. Valid source addresses (aliases) are `event`, `telemetry` and `command_response`. Runtime, these are resolved as following: -* `event` -> `hono.event` -* `telemetry` -> `hono.telemetry` -* `command_response` -> `hono.command_response` +* `event` -> `{%raw%}hono.event.{{connection:id}}{%endraw%}` +* `telemetry` -> `{%raw%}hono.telemetry.{{connection:id}}{%endraw%}` +* `command_response` -> `{%raw%}hono.command_response.{{connection:id}}{%endraw%}` + +Note: The {{connection:id}} will be replaced by the value of connectionId #### Source reply target Similar to source addresses, the reply target `address` is an alias as well. The single valid value for it is `command`. It is resolved to Kafka topic/key like this: -* `command` -> `hono.command/` (<thingId> is substituted by thing ID value). +* `command` -> `{%raw%}hono.command.{{connection:id}}/{%endraw%}` (<thingId> is substituted by thing ID value). + +Note: The {{connection:id}} will be replaced by the value of connectionId The needed header mappings for the `replyTarget` are also populated automatically at runtime and there is no need to specify them in the connection definition. Any of the following specified value will be substituted (i.e. ignored). From 0f093a2c6cec9454bea1ecf044d6c085d4c449a6 Mon Sep 17 00:00:00 2001 From: Abhijeet Mishra Date: Fri, 27 Jan 2023 00:22:23 +0530 Subject: [PATCH 036/173] Added case insensitive search for things-search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Abhijeet Mishra <“abhijeet.mishra498@gmail.com”> --- .../ditto/base/model/common/ILikeHelper.java | 78 +++++++++++++++++++ .../parser/internal/RqlPredicateParser.scala | 10 ++- .../rql/query/criteria/CriteriaFactory.java | 8 ++ .../query/criteria/CriteriaFactoryImpl.java | 9 +++ .../query/criteria/ILikePredicateImpl.java | 43 ++++++++++ .../criteria/visitors/PredicateVisitor.java | 1 + .../filter/ParameterPredicateVisitor.java | 1 + .../ThingPredicatePredicateVisitor.java | 10 ++- .../visitors/CreateBsonPredicateVisitor.java | 12 +++ 9 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java create mode 100644 rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java b/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java new file mode 100644 index 00000000000..04ce7f61c33 --- /dev/null +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java @@ -0,0 +1,78 @@ + +/* + * Copyright (c) 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.base.model.common; + +import java.util.regex.Pattern; + +/** + * A helper to create "like" patterns. + * + * @since 2.3.0 + */ +public final class ILikeHelper { + + private static final String LEADING_WILDCARD = "\\Q\\E.*"; + private static final String TRAILING_WILDCARD = ".*\\Q\\E"; + + private ILikeHelper() { + } + + /** + * Convert a wildcard expression into a regular expression. + * + * The wildcard cards supported are: + * + *
    + *
    *
    Matching any number of any character
    + *
    ?
    Matches any single character
    + *
    + * + * @param expression The wildcard expression to convert. + * @return The regular expression, which can be compiled with {@link Pattern#compile(String)}. + */ + public static String convertToRegexSyntax(final String expression) { + if (expression == null) { + return null; + } + + // simplify expression by replacing repeating wildcard with a single one + final String valueString = replaceRepeatingWildcards(expression); + // shortcut for single * wildcard + if ("*".equals(valueString)) { + return ".*"; + } + + // first escape the whole string + String escapedString = Pattern.compile(Pattern.quote(valueString)).toString(); + // then enable allowed wildcards (* and ?) again + escapedString = escapedString.replaceAll("\\*", "\\\\E.*\\\\Q"); + escapedString = escapedString.replaceAll("\\?", "\\\\E.\\\\Q"); // escape Char wild cards for ? + + // prepend ^ if is a prefix match (no * at the beginning of the string, much faster) + if (!valueString.startsWith(LEADING_WILDCARD)) { + escapedString = "^" + escapedString; + } + // append $ if is a postfix match (no * at the end of the string, much faster) + if (!valueString.endsWith(TRAILING_WILDCARD)) { + escapedString = escapedString + "$"; + } + return escapedString; + } + + private static String replaceRepeatingWildcards(final String value) { + return value.replaceAll("\\*{2,}", "*"); + } + +} + diff --git a/rql/parser/src/main/scala/org/eclipse/ditto/rql/parser/internal/RqlPredicateParser.scala b/rql/parser/src/main/scala/org/eclipse/ditto/rql/parser/internal/RqlPredicateParser.scala index b0312f2a718..4da7f37112e 100644 --- a/rql/parser/src/main/scala/org/eclipse/ditto/rql/parser/internal/RqlPredicateParser.scala +++ b/rql/parser/src/main/scala/org/eclipse/ditto/rql/parser/internal/RqlPredicateParser.scala @@ -28,7 +28,7 @@ import scala.util.{Failure, Success} *
       * Query                      = SingleComparisonOp | MultiComparisonOp | MultiLogicalOp | SingleLogicalOp | ExistsOp
       * SingleComparisonOp         = SingleComparisonName, '(', ComparisonProperty, ',', ComparisonValue, ')'
    -  * SingleComparisonName       = "eq" | "ne" | "gt" | "ge" | "lt" | "le" | "like"
    +  * SingleComparisonName       = "eq" | "ne" | "gt" | "ge" | "lt" | "le" | "like" | "ilike"
       * MultiComparisonOp          = MultiComparisonName, '(', ComparisonProperty, ',', ComparisonValue, { ',', ComparisonValue }, ')'
       * MultiComparisonName        = "in"
       * MultiLogicalOp             = MultiLogicalName, '(', Query, { ',', Query }, ')'
    @@ -71,10 +71,10 @@ private class RqlPredicateParser(override val input: ParserInput) extends RqlPar
       }
     
       /**
    -    * SingleComparisonName       = "eq" | "ne" | "gt" | "ge" | "lt" | "le" | "like"
    +    * SingleComparisonName       = "eq" | "ne" | "gt" | "ge" | "lt" | "le" | "like" | "ilike"
         */
       private def SingleComparisonName: Rule1[SingleComparisonNode.Type] = rule {
    -    eq | ne | gt | ge | lt | le | like
    +    eq | ne | gt | ge | lt | le | like | ilike
       }
     
       private def eq: Rule1[SingleComparisonNode.Type] = rule {
    @@ -105,6 +105,10 @@ private class RqlPredicateParser(override val input: ParserInput) extends RqlPar
         "like" ~ push(SingleComparisonNode.Type.LIKE)
       }
     
    +  private def ilike: Rule1[SingleComparisonNode.Type] = rule {
    +    "ilike" ~ push(SingleComparisonNode.Type.ILIKE)
    +  }
    +
       /**
         * MultiComparisonOp          = MultiComparisonName, '(', ComparisonProperty, ',', ComparisonValue, { ',', ComparisonValue }, ')'
         */
    diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java
    index cc78834d5ae..74f4641c411 100644
    --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java
    +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java
    @@ -152,6 +152,14 @@ default Criteria nor(final Criteria criteria) {
          */
         Predicate like(@Nullable Object value);
     
    +    /**
    +     *Creates a predicate which checks lower than or equals.
    +     *
    +     * @param value the value, may be {@code null}.
    +     * @return the predicate. 
    +     */
    +    Predicate ilike(@Nullable object value);
    +    
         /**
          * The $in predicate selects the documents where the value of a field equals any value in the specified array.
          *
    diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java
    index b1a195c62a0..e825b0b2526 100644
    --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java
    +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java
    @@ -110,6 +110,15 @@ public Predicate like(@Nullable final Object value) {
             }
         }
     
    +    @Override
    +    public Predicate ilike(@Nullable final Object value) {
    +        if (value instanceof String) {
    +            return new ILikePredicateImpl(value);
    +        } else {
    +            throw new IllegalArgumentException("In the like predicate only string values are allowed.");
    +        }
    +    }
    +
         @Override
         public Predicate in(final List values) {
             return new InPredicateImpl(requireNonNull(values));
    diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java
    new file mode 100644
    index 00000000000..fbf55977a77
    --- /dev/null
    +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java
    @@ -0,0 +1,43 @@
    +/*
    + * Copyright (c) 2017 Contributors to the Eclipse Foundation
    + *
    + * See the NOTICE file(s) distributed with this work for additional
    + * information regarding copyright ownership.
    + *
    + * This program and the accompanying materials are made available under the
    + * terms of the Eclipse Public License 2.0 which is available at
    + * http://www.eclipse.org/legal/epl-2.0
    + *
    + * SPDX-License-Identifier: EPL-2.0
    + */
    +package org.eclipse.ditto.rql.query.criteria;
    +
    +import javax.annotation.Nullable;
    +
    +import org.eclipse.ditto.base.model.common.ILikeHelper;
    +import org.eclipse.ditto.rql.query.criteria.visitors.PredicateVisitor;
    +
    +/**
    + * ILike predicate.
    + */
    +final class ILikePredicateImpl extends AbstractSinglePredicate {
    +
    +    public ILikePredicateImpl(@Nullable final Object value) {
    +        super(value);
    +    }
    +
    +    @Nullable
    +    private String convertToRegexSyntaxAndGetOption() {
    +        final Object value = getValue();
    +        if (value != null) {
    +            return ILikeHelper.convertToRegexSyntax(value.toString());
    +        } else {
    +            return null;
    +        }
    +    }
    +
    +    @Override
    +    public  T accept(final PredicateVisitor visitor) {
    +        return visitor.visitILike(convertToRegexSyntaxAndGetOption());
    +    }
    +}
    diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java
    index 868d2245ed4..1b42b856b10 100644
    --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java
    +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java
    @@ -39,4 +39,5 @@ public interface PredicateVisitor {
     
         T visitIn(List values);
     
    +    T visitILike(@Nullable String value)
     }
    diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/filter/ParameterPredicateVisitor.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/filter/ParameterPredicateVisitor.java
    index a73b7ec2ad9..90acb7e0b9b 100644
    --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/filter/ParameterPredicateVisitor.java
    +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/filter/ParameterPredicateVisitor.java
    @@ -54,6 +54,7 @@ final class ParameterPredicateVisitor implements PredicateVisitor {
             SINGLE_COMPARISON_NODE_MAPPING.put(SingleComparisonNode.Type.LT, CriteriaFactory::lt);
             SINGLE_COMPARISON_NODE_MAPPING.put(SingleComparisonNode.Type.LE, CriteriaFactory::le);
             SINGLE_COMPARISON_NODE_MAPPING.put(SingleComparisonNode.Type.LIKE, CriteriaFactory::like);
    +        SINGLE_COMPARISON_NODE_MAPPING.put(SingleComparisonNode.Type.ILIKE, CriteriaFactory::ilike);
         }
     
         private final List criteria = new ArrayList<>();
    diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java
    index fdcda2c60a6..76a4f1fad5a 100644
    --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java
    +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java
    @@ -240,7 +240,15 @@ public Function> visitLike(@Nullable final String value
                             .filter(str -> null != value && Pattern.compile(value).matcher(str).matches())
                             .isPresent();
         }
    -
    +    @Override
    +    public Function> visitILike(@Nullable final String value) {
    +        return fieldName ->
    +                thing -> getThingField(fieldName, thing)
    +                        .filter(JsonValue::isString)
    +                        .map(JsonValue::asString)
    +                        .filter(str -> null != value && Pattern.compile(value, Pattern.CASE_INSENSITIVE).matcher(str).matches())
    +                        .isPresent();
    +    }
         @Nullable
         private Object resolveValue(@Nullable final Object value) {
             if (value instanceof ParsedPlaceholder) {
    diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java
    index 22bf085b85a..a8ebc60753b 100644
    --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java
    +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java
    @@ -137,6 +137,18 @@ public Function visitLike(final String value) {
             return fieldName -> Filters.regex(fieldName, valueWithoutLeadingOrTrailingWildcard, "");
         }
     
    +    @Override
    +    public Function visitILike(final String value) {
    +        // remove leading or trailing wildcard because queries like /^a/ are much faster than /^a.*$/ or /^a.*/
    +        // from mongodb docs:
    +        // "Additionally, while /^a/, /^a.*/, and /^a.*$/ match equivalent strings, they have different performance
    +        // characteristics. All of these expressions use an index if an appropriate index exists;
    +        // however, /^a.*/, and /^a.*$/ are slower. /^a/ can stop scanning after matching the prefix."
    +        final String valueWithoutLeadingOrTrailingWildcard = removeLeadingOrTrailingWildcard(value);
    +        Pattern pattern = Pattern.compile(valueWithoutLeadingOrTrailingWildcard, Pattern.CASE_INSENSITIVE)
    +        return fieldName -> Filters.regex(fieldName, pattern, "");
    +    }
    +
         private static String removeLeadingOrTrailingWildcard(final String valueString) {
             String valueWithoutLeadingOrTrailingWildcard = valueString;
             if (valueString.startsWith(LEADING_WILDCARD)) {
    
    From 9fae3bf289209d663f993a1adafed4d85a0b1b9d Mon Sep 17 00:00:00 2001
    From: Abhijeet Mishra 
    Date: Sun, 5 Feb 2023 21:26:35 +0530
    Subject: [PATCH 037/173] correction & update the year in the copyright headers
     of the added files to 2023
    MIME-Version: 1.0
    Content-Type: text/plain; charset=UTF-8
    Content-Transfer-Encoding: 8bit
    
    Signed-off-by: Abhijeet Mishra <“abhijeet.mishra498@gmail.com”>
    ---
     .../ditto/base/model/common/ILikeHelper.java  |  2 +-
     .../base/model/common/ILikeHelperTest.java    | 38 +++++++++++++++++++
     .../predicates/ast/SingleComparisonNode.java  |  7 +++-
     .../rql/query/criteria/CriteriaFactory.java   |  4 +-
     .../query/criteria/ILikePredicateImpl.java    |  2 +-
     .../criteria/visitors/PredicateVisitor.java   |  2 +-
     .../visitors/CreateBsonPredicateVisitor.java  |  5 ++-
     7 files changed, 52 insertions(+), 8 deletions(-)
     create mode 100644 base/model/src/test/java/org/eclipse/ditto/base/model/common/ILikeHelperTest.java
    
    diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java b/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java
    index 04ce7f61c33..6f118c1f128 100644
    --- a/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java
    +++ b/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java
    @@ -1,6 +1,6 @@
     
     /*
    - * Copyright (c) 2022 Contributors to the Eclipse Foundation
    + * Copyright (c) 2023 Contributors to the Eclipse Foundation
      *
      * See the NOTICE file(s) distributed with this work for additional
      * information regarding copyright ownership.
    diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/common/ILikeHelperTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/common/ILikeHelperTest.java
    new file mode 100644
    index 00000000000..a5d68283bf0
    --- /dev/null
    +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/common/ILikeHelperTest.java
    @@ -0,0 +1,38 @@
    +/*
    + * Copyright (c) 2023 Contributors to the Eclipse Foundation
    + *
    + * See the NOTICE file(s) distributed with this work for additional
    + * information regarding copyright ownership.
    + *
    + * This program and the accompanying materials are made available under the
    + * terms of the Eclipse Public License 2.0 which is available at
    + * http://www.eclipse.org/legal/epl-2.0
    + *
    + * SPDX-License-Identifier: EPL-2.0
    + */
    +package org.eclipse.ditto.base.model.common;
    +
    +import java.util.regex.Pattern;
    +
    +import org.junit.Assert;
    +import org.junit.Test;
    +
    +public final class ILikeHelperTest {
    +
    +    @Test
    +    public void testWildcards() {
    +        assertExpression("", "*", true);
    +        assertExpression("foo", "*", true);
    +        assertExpression("foo.bar", "FOO.BAR", true);
    +        assertExpression("foo..bar", "foo.bar", false);
    +        assertExpression("foo..bar", "FOO*", true);
    +        assertExpression("foo..bar", "*Bar", true);
    +        assertExpression("foo.bar.baz", "bar", false);
    +        assertExpression("foo.bar.baz", "*bAr*", true);
    +    }
    +
    +    private static void assertExpression(final String value, final String expression, final boolean matches) {
    +        Pattern p = Pattern.compile(LikeHelper.convertToRegexSyntax(expression), Pattern.CASE_INSENSITIVE);
    +        Assert.assertEquals(matches, p.matcher(value).matches());
    +    }
    +}
    diff --git a/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java b/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java
    index b8359d0cc90..70b492c7fb2 100755
    --- a/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java
    +++ b/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java
    @@ -78,7 +78,12 @@ public enum Type {
             /**
              * Represents a lower than or equals comparison.
              */
    -        LIKE("like");
    +        LIKE("like"),
    +        
    +         /**
    +         * Represents a lower than or equals comparison case insensitive.
    +         */
    +        ILIKE("ilike");
     
             private final String name;
     
    diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java
    index 74f4641c411..f397746406c 100644
    --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java
    +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java
    @@ -153,12 +153,12 @@ default Criteria nor(final Criteria criteria) {
         Predicate like(@Nullable Object value);
     
         /**
    -     *Creates a predicate which checks lower than or equals.
    +     *Creates a predicate which checks lower than or equals case insensitive.
          *
          * @param value the value, may be {@code null}.
          * @return the predicate. 
          */
    -    Predicate ilike(@Nullable object value);
    +    Predicate ilike(@Nullable Object value);
         
         /**
          * The $in predicate selects the documents where the value of a field equals any value in the specified array.
    diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java
    index fbf55977a77..fc915438975 100644
    --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java
    +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java
    @@ -1,5 +1,5 @@
     /*
    - * Copyright (c) 2017 Contributors to the Eclipse Foundation
    + * Copyright (c) 2023 Contributors to the Eclipse Foundation
      *
      * See the NOTICE file(s) distributed with this work for additional
      * information regarding copyright ownership.
    diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java
    index 1b42b856b10..68e732efda5 100644
    --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java
    +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java
    @@ -39,5 +39,5 @@ public interface PredicateVisitor {
     
         T visitIn(List values);
     
    -    T visitILike(@Nullable String value)
    +    T visitILike(@Nullable String value);
     }
    diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java
    index a8ebc60753b..09154ae9e54 100644
    --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java
    +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonPredicateVisitor.java
    @@ -19,6 +19,7 @@
     import java.util.List;
     import java.util.function.Function;
     import java.util.stream.Collectors;
    +import java.util.regex.Pattern;
     
     import javax.annotation.Nullable;
     
    @@ -145,8 +146,8 @@ public Function visitILike(final String value) {
             // characteristics. All of these expressions use an index if an appropriate index exists;
             // however, /^a.*/, and /^a.*$/ are slower. /^a/ can stop scanning after matching the prefix."
             final String valueWithoutLeadingOrTrailingWildcard = removeLeadingOrTrailingWildcard(value);
    -        Pattern pattern = Pattern.compile(valueWithoutLeadingOrTrailingWildcard, Pattern.CASE_INSENSITIVE)
    -        return fieldName -> Filters.regex(fieldName, pattern, "");
    +        Pattern pattern = Pattern.compile(valueWithoutLeadingOrTrailingWildcard, Pattern.CASE_INSENSITIVE);
    +        return fieldName -> Filters.regex(fieldName, pattern);
         }
     
         private static String removeLeadingOrTrailingWildcard(final String valueString) {
    
    From 9e9af25714b07e05a3995cd1685956e1ef65d7d0 Mon Sep 17 00:00:00 2001
    From: Abhijeet Mishra 
    Date: Mon, 13 Feb 2023 18:36:38 +0530
    Subject: [PATCH 038/173] correction and refactoring in ilike
    MIME-Version: 1.0
    Content-Type: text/plain; charset=UTF-8
    Content-Transfer-Encoding: 8bit
    
    Signed-off-by: Abhijeet Mishra <“abhijeet.mishra498@gmail.com”>
    ---
     .../ditto/base/model/common/ILikeHelper.java  | 78 -------------------
     .../base/model/common/ILikeHelperTest.java    | 38 ---------
     .../base/model/common/LikeHelperTest.java     | 10 +++
     .../sources/parameters/searchFilter.yml       |  2 +
     .../main/resources/pages/ditto/basic-rql.md   | 32 ++++++++
     .../predicates/ast/SingleComparisonNode.java  |  4 +-
     .../rql/query/criteria/CriteriaFactory.java   |  6 +-
     .../query/criteria/CriteriaFactoryImpl.java   |  2 +-
     .../query/criteria/ILikePredicateImpl.java    |  4 +-
     .../criteria/visitors/PredicateVisitor.java   |  3 +-
     .../ThingPredicatePredicateVisitor.java       |  2 +
     .../ThingPredicatePredicateVisitorTest.java   |  7 ++
     12 files changed, 63 insertions(+), 125 deletions(-)
     delete mode 100644 base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java
     delete mode 100644 base/model/src/test/java/org/eclipse/ditto/base/model/common/ILikeHelperTest.java
    
    diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java b/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java
    deleted file mode 100644
    index 6f118c1f128..00000000000
    --- a/base/model/src/main/java/org/eclipse/ditto/base/model/common/ILikeHelper.java
    +++ /dev/null
    @@ -1,78 +0,0 @@
    -
    -/*
    - * Copyright (c) 2023 Contributors to the Eclipse Foundation
    - *
    - * See the NOTICE file(s) distributed with this work for additional
    - * information regarding copyright ownership.
    - *
    - * This program and the accompanying materials are made available under the
    - * terms of the Eclipse Public License 2.0 which is available at
    - * http://www.eclipse.org/legal/epl-2.0
    - *
    - * SPDX-License-Identifier: EPL-2.0
    - */
    -package org.eclipse.ditto.base.model.common;
    -
    -import java.util.regex.Pattern;
    -
    -/**
    - * A helper to create "like" patterns.
    - *
    - * @since 2.3.0
    - */
    -public final class ILikeHelper {
    -
    -    private static final String LEADING_WILDCARD = "\\Q\\E.*";
    -    private static final String TRAILING_WILDCARD = ".*\\Q\\E";
    -
    -    private ILikeHelper() {
    -    }
    -
    -    /**
    -     * Convert a wildcard expression into a regular expression.
    -     *
    -     * The wildcard cards supported are:
    -     *
    -     * 
    - *
    *
    Matching any number of any character
    - *
    ?
    Matches any single character
    - *
    - * - * @param expression The wildcard expression to convert. - * @return The regular expression, which can be compiled with {@link Pattern#compile(String)}. - */ - public static String convertToRegexSyntax(final String expression) { - if (expression == null) { - return null; - } - - // simplify expression by replacing repeating wildcard with a single one - final String valueString = replaceRepeatingWildcards(expression); - // shortcut for single * wildcard - if ("*".equals(valueString)) { - return ".*"; - } - - // first escape the whole string - String escapedString = Pattern.compile(Pattern.quote(valueString)).toString(); - // then enable allowed wildcards (* and ?) again - escapedString = escapedString.replaceAll("\\*", "\\\\E.*\\\\Q"); - escapedString = escapedString.replaceAll("\\?", "\\\\E.\\\\Q"); // escape Char wild cards for ? - - // prepend ^ if is a prefix match (no * at the beginning of the string, much faster) - if (!valueString.startsWith(LEADING_WILDCARD)) { - escapedString = "^" + escapedString; - } - // append $ if is a postfix match (no * at the end of the string, much faster) - if (!valueString.endsWith(TRAILING_WILDCARD)) { - escapedString = escapedString + "$"; - } - return escapedString; - } - - private static String replaceRepeatingWildcards(final String value) { - return value.replaceAll("\\*{2,}", "*"); - } - -} - diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/common/ILikeHelperTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/common/ILikeHelperTest.java deleted file mode 100644 index a5d68283bf0..00000000000 --- a/base/model/src/test/java/org/eclipse/ditto/base/model/common/ILikeHelperTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ -package org.eclipse.ditto.base.model.common; - -import java.util.regex.Pattern; - -import org.junit.Assert; -import org.junit.Test; - -public final class ILikeHelperTest { - - @Test - public void testWildcards() { - assertExpression("", "*", true); - assertExpression("foo", "*", true); - assertExpression("foo.bar", "FOO.BAR", true); - assertExpression("foo..bar", "foo.bar", false); - assertExpression("foo..bar", "FOO*", true); - assertExpression("foo..bar", "*Bar", true); - assertExpression("foo.bar.baz", "bar", false); - assertExpression("foo.bar.baz", "*bAr*", true); - } - - private static void assertExpression(final String value, final String expression, final boolean matches) { - Pattern p = Pattern.compile(LikeHelper.convertToRegexSyntax(expression), Pattern.CASE_INSENSITIVE); - Assert.assertEquals(matches, p.matcher(value).matches()); - } -} diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java index 41d134522dc..1c13f78a245 100644 --- a/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java @@ -21,6 +21,7 @@ public final class LikeHelperTest { @Test public void testWildcards() { + //case sensitive test cases assertExpression("", "*", true); assertExpression("foo", "*", true); assertExpression("foo.bar", "foo.bar", true); @@ -29,6 +30,15 @@ public void testWildcards() { assertExpression("foo..bar", "*bar", true); assertExpression("foo.bar.baz", "bar", false); assertExpression("foo.bar.baz", "*bar*", true); + //case insensitive test cases appending this (?i) before text + assertExpression("", "*", true); + assertExpression("foo", "*", true); + assertExpression("foo.bar", "(?i)FOO.BAR", true); + assertExpression("foo..bar", "(?i)foo.bar", false); + assertExpression("foo..bar", "(?i)FOO*", true); + assertExpression("foo..bar", "*(?i)Bar", true); + assertExpression("foo.bar.baz", "(?i)bar", false); + assertExpression("foo.bar.baz", "*(?i)bAr*", true); } private static void assertExpression(final String value, final String expression, final boolean matches) { diff --git a/documentation/src/main/resources/openapi/sources/parameters/searchFilter.yml b/documentation/src/main/resources/openapi/sources/parameters/searchFilter.yml index cc62a721e5b..6dbd6a803d4 100644 --- a/documentation/src/main/resources/openapi/sources/parameters/searchFilter.yml +++ b/documentation/src/main/resources/openapi/sources/parameters/searchFilter.yml @@ -30,6 +30,8 @@ description: |- * ```like({property},{value})``` (i.e. contains values similar to the expressions listed) + * ```ilike({property},{value})``` (i.e. contains values similar and case insensitive to the expressions listed) + * ```exists({property})``` (i.e. all things in which the given path exists) diff --git a/documentation/src/main/resources/pages/ditto/basic-rql.md b/documentation/src/main/resources/pages/ditto/basic-rql.md index 23fcc0acb37..d3897dec88d 100644 --- a/documentation/src/main/resources/pages/ditto/basic-rql.md +++ b/documentation/src/main/resources/pages/ditto/basic-rql.md @@ -218,6 +218,38 @@ like(attributes/key1,"*known-chars-in-between*") like(attributes/key1,"just-som?-char?-unkn?wn") ``` +#### ilike +Filter property values which are like (similar) and case insensitive ``. + +``` +ilike(,) +``` + +{% include note.html content="The `ilike` operator is not defined in the linked RQL grammar, it is a Ditto + specific operator." %} + +**Details concerning the ilike-operator** + +The `ilike` operator provides some regular expression capabilities for pattern matching Strings with case insensitivity. + +The following expressions are supported: + +* \*endswith => match at the end of a specific String. +* startswith\* => match at the beginning of a specific String. +* \*contains\* => match if contains a specific String. +* Th?ng => match for a wildcard character. + +**Examples** +``` +ilike(attributes/key1,"*known-CHARS-at-end") + +ilike(attributes/key1,"known-chars-AT-start*") + +ilike(attributes/key1,"*KNOWN-CHARS-IN-BETWEEN*") + +ilike(attributes/key1,"just-som?-char?-unkn?wn") +``` + #### exists Filter property values which exist. diff --git a/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java b/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java index 70b492c7fb2..421947b8536 100755 --- a/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java +++ b/rql/model/src/main/java/org/eclipse/ditto/rql/model/predicates/ast/SingleComparisonNode.java @@ -76,12 +76,12 @@ public enum Type { LE("le"), /** - * Represents a lower than or equals comparison. + * Represents a string 'like' comparison, supporting wildcards '*' for multiple and '?' for a single character. */ LIKE("like"), /** - * Represents a lower than or equals comparison case insensitive. + * Represents a string 'like' comparison, supporting wildcards '*' for multiple and '?' for a single character with case sensitivity. */ ILIKE("ilike"); diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java index f397746406c..f27a67e2a6e 100644 --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactory.java @@ -145,7 +145,7 @@ default Criteria nor(final Criteria criteria) { Predicate le(@Nullable Object value); /** - * Creates a predicate which checks lower than or equals. + * Represents a string 'like' comparison, supporting wildcards '*' for multiple and '?' for a single character. * * @param value the value, may be {@code null}. * @return the predicate. @@ -153,8 +153,8 @@ default Criteria nor(final Criteria criteria) { Predicate like(@Nullable Object value); /** - *Creates a predicate which checks lower than or equals case insensitive. - * + * Represents a string 'like' comparison, supporting wildcards '*' for multiple and '?' for a single character with case insensitivity. + * @since 3.2.0 * @param value the value, may be {@code null}. * @return the predicate. */ diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java index e825b0b2526..02de81067c2 100644 --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/CriteriaFactoryImpl.java @@ -115,7 +115,7 @@ public Predicate ilike(@Nullable final Object value) { if (value instanceof String) { return new ILikePredicateImpl(value); } else { - throw new IllegalArgumentException("In the like predicate only string values are allowed."); + throw new IllegalArgumentException("In the ilike predicate only string values are allowed."); } } diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java index fc915438975..19cd596252c 100644 --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/ILikePredicateImpl.java @@ -14,7 +14,7 @@ import javax.annotation.Nullable; -import org.eclipse.ditto.base.model.common.ILikeHelper; +import org.eclipse.ditto.base.model.common.LikeHelper; import org.eclipse.ditto.rql.query.criteria.visitors.PredicateVisitor; /** @@ -30,7 +30,7 @@ public ILikePredicateImpl(@Nullable final Object value) { private String convertToRegexSyntaxAndGetOption() { final Object value = getValue(); if (value != null) { - return ILikeHelper.convertToRegexSyntax(value.toString()); + return LikeHelper.convertToRegexSyntax(value.toString()); } else { return null; } diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java index 68e732efda5..cc016f33594 100644 --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/criteria/visitors/PredicateVisitor.java @@ -37,7 +37,8 @@ public interface PredicateVisitor { T visitLike(@Nullable String value); + T visitILike(@Nullable String value); + T visitIn(List values); - T visitILike(@Nullable String value); } diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java index 76a4f1fad5a..3fef861ef91 100644 --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitor.java @@ -240,6 +240,7 @@ public Function> visitLike(@Nullable final String value .filter(str -> null != value && Pattern.compile(value).matcher(str).matches()) .isPresent(); } + @Override public Function> visitILike(@Nullable final String value) { return fieldName -> @@ -249,6 +250,7 @@ public Function> visitILike(@Nullable final String valu .filter(str -> null != value && Pattern.compile(value, Pattern.CASE_INSENSITIVE).matcher(str).matches()) .isPresent(); } + @Nullable private Object resolveValue(@Nullable final Object value) { if (value instanceof ParsedPlaceholder) { diff --git a/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java b/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java index 691f740d3c9..0a036c4d33b 100644 --- a/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java +++ b/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java @@ -137,6 +137,13 @@ public void matchingStringLike() { .isTrue(); } + @Test + public void matchingStringILike() { + // the sut already works on regex Pattern - the translation from "*" to ".*" followed by case insensitivity is done in LikePredicateImpl + doTest(sut.visitLike("this-is.*"), JsonValue.of("THIS-IS-THE-CONTENT")) + .isTrue(); + } + @Test public void matchingViaPlaceholderStringLike() { // the sut already works on regex Pattern - the translation from "*" to ".*" is done in LikePredicateImpl From 877532a804c083df2768affcc5d2316cac598526 Mon Sep 17 00:00:00 2001 From: Abhijeet Mishra Date: Tue, 14 Feb 2023 15:49:58 +0530 Subject: [PATCH 039/173] correction & refactoring in LikeHelperTest class & removed copy&paste error Signed-off-by: Abhijeet Mishra --- .../base/model/common/LikeHelperTest.java | 28 +++++++++++++------ .../ThingPredicatePredicateVisitorTest.java | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java b/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java index 1c13f78a245..be15e71c7f9 100644 --- a/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java +++ b/base/model/src/test/java/org/eclipse/ditto/base/model/common/LikeHelperTest.java @@ -30,19 +30,29 @@ public void testWildcards() { assertExpression("foo..bar", "*bar", true); assertExpression("foo.bar.baz", "bar", false); assertExpression("foo.bar.baz", "*bar*", true); - //case insensitive test cases appending this (?i) before text - assertExpression("", "*", true); - assertExpression("foo", "*", true); - assertExpression("foo.bar", "(?i)FOO.BAR", true); - assertExpression("foo..bar", "(?i)foo.bar", false); - assertExpression("foo..bar", "(?i)FOO*", true); - assertExpression("foo..bar", "*(?i)Bar", true); - assertExpression("foo.bar.baz", "(?i)bar", false); - assertExpression("foo.bar.baz", "*(?i)bAr*", true); } private static void assertExpression(final String value, final String expression, final boolean matches) { Pattern p = Pattern.compile(LikeHelper.convertToRegexSyntax(expression)); Assert.assertEquals(matches, p.matcher(value).matches()); } + + @Test + public void testCaseInsensitiveWildcards() { + //case insensitive test cases + assertExpressionCaseInsensitive("", "*", true); + assertExpressionCaseInsensitive("foo", "*", true); + assertExpressionCaseInsensitive("foo.bar", "FOO.BAR", true); + assertExpressionCaseInsensitive("foo..bar", "foo.bar", false); + assertExpressionCaseInsensitive("foo..bar", "FOO*", true); + assertExpressionCaseInsensitive("foo..bar", "*Bar", true); + assertExpressionCaseInsensitive("foo.bar.baz", "bar", false); + assertExpressionCaseInsensitive("foo.bar.baz", "*bAr*", true); + } + + private static void assertExpressionCaseInsensitive(final String value, final String expression, final boolean matches) { + Pattern p = Pattern.compile(LikeHelper.convertToRegexSyntax(expression), Pattern.CASE_INSENSITIVE); + Assert.assertEquals(matches, p.matcher(value).matches()); + } + } diff --git a/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java b/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java index 0a036c4d33b..c27423ae570 100644 --- a/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java +++ b/rql/query/src/test/java/org/eclipse/ditto/rql/query/things/ThingPredicatePredicateVisitorTest.java @@ -140,7 +140,7 @@ public void matchingStringLike() { @Test public void matchingStringILike() { // the sut already works on regex Pattern - the translation from "*" to ".*" followed by case insensitivity is done in LikePredicateImpl - doTest(sut.visitLike("this-is.*"), JsonValue.of("THIS-IS-THE-CONTENT")) + doTest(sut.visitILike("this-is.*"), JsonValue.of("THIS-IS-THE-CONTENT")) .isTrue(); } From f47d440d23ab7d7a4ce01fadea41ee67cca4bd1e Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Tue, 14 Feb 2023 12:02:37 +0100 Subject: [PATCH 040/173] stabilize MQTT unsubscribing Signed-off-by: Thomas Jaeckle --- .../client/BaseGenericMqttSubscribingClient.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/client/BaseGenericMqttSubscribingClient.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/client/BaseGenericMqttSubscribingClient.java index 7d1f35158b6..4ec4fc9ad48 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/client/BaseGenericMqttSubscribingClient.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/mqtt/hivemq/client/BaseGenericMqttSubscribingClient.java @@ -226,9 +226,13 @@ protected Single sendSubscribe(final Mqtt3RxClient mqtt3RxCli @Override Completable sendUnsubscribe(final Mqtt3RxClient mqtt3RxClient, final MqttTopicFilter... mqttTopicFilters) { - final var unsubscribe = - Mqtt3Unsubscribe.builder().addTopicFilters(mqttTopicFilters).build(); - return mqtt3RxClient.unsubscribe(unsubscribe); + if (mqttTopicFilters.length == 0) { + return Completable.complete(); + } else { + final var unsubscribe = + Mqtt3Unsubscribe.builder().addTopicFilters(mqttTopicFilters).build(); + return mqtt3RxClient.unsubscribe(unsubscribe); + } } @Override From 68a5e764b6e2a145c8093e41b3721c549f781628 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Tue, 14 Feb 2023 14:06:38 +0100 Subject: [PATCH 041/173] added agreed on "additional OSS contributing rules" for Eclipse Ditto to existing CONTRIBUTING.md Signed-off-by: Thomas Jaeckle --- CONTRIBUTING.md | 137 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 127 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1cb0974a9f3..09328574e5c 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,7 @@ Please make sure any file you newly create contains a proper license header. Fin Adjusted for Java classes: ```java /* - * Copyright (c) 2019 Contributors to the Eclipse Foundation + * Copyright (c) 2023 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -72,7 +72,7 @@ Adjusted for Java classes: Adjusted for XML files: ```xml ``` -### Important - -Please do not forget to add your name/organization to the [legal/NOTICE.md](legal/NOTICE.md) file's Copyright Holders -section. If this is not the first contribution you make, then simply update the time period contained in the copyright -entry to use the year of your first contribution as the lower boundary and the current year as the upper boundary, e.g. - - Copyright 2018-2019 ACME Corporation - ## Submitting the Changes Submit a pull request via the normal GitHub UI. @@ -100,3 +92,128 @@ Submit a pull request via the normal GitHub UI. ## After Submitting * Do not use your branch for any other development, otherwise further changes that you make will be visible in the PR. + + +# OSS development process - rules + +As of 02/2023, the following additional "rules" regarding the open OSS development process were agreed on. + +## Addition of new features via feature toggles + +Goals: +* Reduce the risk for other Ditto users that a new feature have an impact on existing functionality and stability +* Whenever possible (and feasible), added functionality shall be added using a "feature toggle". + +Ditto already has a class `FeatureToggle.java` where feature toggles are contained and providing functionality to +"secure" a feature with a method in there which throws an `UnsupportedSignalException` once a feature is used which +is disabled via feature toggle. + +The toggles are then configured in `ditto-devops.conf` file and can be enabled/disabled via the contained environment variables. + +## Creating GitHub issues before starting to work on code + +Goals: +* Improve transparency on what is currently happening +* Openly discuss new features and whether they are a good fit for Ditto +* Reduce "time waste" + +Whenever a new feature or a bugfix is being worked on, we want to create an issue in Ditto's GitHub project **beforehand**: +https://github.com/eclipse-ditto/ditto/issues + +This provides the needed transparency for other contributors before much effort is put into a new topic in order to: +* Get input on the background (e.g. use case behind / "the need") of the feature/bugfix +* Provide feedback, maybe even suggesting alternatives instead +* Provide suggestions of how to implement it the most efficient way +* Maybe even find synergies when more than 1 contributing companies currently have the same topic to work on + +The following situation shall be prevented: +* If no issue is created upfront, a contributing company e.g. invests 2 months of work in a new feature +* Then a PR is created with this new functionality +* Only then, a discussion with other contributors can start +* At this point, when there e.g. is a big flaw in the architecture or security or API stability of the added functionality, + the invested 2 months could - in the worst case - be a complete waste of time +* This could easily be resolved by discussing it beforehand + +## Create PullRequests early + +Goals: +* Get early feedback on implementation details of new features / bugfixes +* Prevent that an implementation goes "into the wrong direction" (e.g. performance or security wise) + +PullRequests should be created quite early and publicly on the Ditto project. +If they are not yet "ready" to review/merge, they must be marked as "DRAFT" - once they are ready, they can be marked +as such and a review can be performed. + +## Make use of GitHub "Projects" for showing current work for next release + +Goals: +* Make transparent "who" currently works on "what" +* Make transparent what the current agenda for the next Ditto release is + +The new "Projects" capabilities of GitHub look more than sufficient of what we want to achieve here: +* https://github.com/orgs/eclipse-ditto/projects/1 +* https://github.com/orgs/eclipse-ditto/projects/1/views/2 (table view is especially useful, as grouping by "Milestone" is necessary) + +## Establish system-tests in the OpenSource codebase + +Goals: +* Provide means to run automated tests for future enhancements to Ditto +* Secure existing functionality, avoid breaking APIs and existing functionality when changes to the Ditto OSS codebase are done + +The system tests for Eclipse Ditto were initiated here: +https://github.com/eclipse-ditto/ditto-testing + +The tests should be part of the validations done in a PR before a PR is approved and merged. +In order to be able to do that, we want to clarify if the Eclipse Foundation can provide enough resources in order to +run the system-tests in a stable way. + +Currently, that seems to be quite difficult, as projects only have very limited resources in order to run their builds. +In addition, the CI runs in an OpenShift cluster with additional restrictions, e.g. regarding the kind of Docker images +which can be run, exposing of the Docker socket, etc. + +## Regular community meetings + +Goals: + +* Discuss upcoming topics/problems in advance +* Stay in touch via audio/video +* Build up a (contributor and adopter) community who can help each other + +We want to re-establish regular community meetings/call, e.g. a meeting every 2 weeks for 1 hour. +We can utilize the Zoom account from the Eclipse Foundation to have a "neutral" one .. or just use "Google Meet". + +## Chat for (internal) exchanges + +Goals: +* Have a direct channel where to reach other Ditto committers and contributors +* In order to get timely responses if e.g. a bugfix release has to be scheduled/done quickly + +We can use "Gitter.IM" communities to add different rooms of which some also can be private: +https://gitter.im/EclipseDitto/community + +## Release strategy + +Goals: +* Have rules of how often to do "planned feature releases" +* Have options for contributing companies to prioritize a release (e.g. if urgent bugfix or urgent feature release) + +The suggestion would be to have approximately 4 planned minor releases per year, 1 each quarter (e.g. 03 / 06 / 09 / 12). +If needed and all contributing companies agree minor releases can also happen earlier/more often. + +Bugfix releases should be done immediately if a critical bug was fixed and either the contributors or the community need a quick fix release. + +## Approving / merging PRs + +Goals: +* PullRequests - once they are ready - shall not stay unmerged for a long time as this leads to the risk they are not + mergable or get outdated quickly + +Approach: + +* Before merging a PR at least 1 approval is required + * Approvals shall only be issued after a code review + * Preferably that would be 1 approval from an existing Ditto committer + * But could also be the approval of an active contributor who does not yet have committer status +* If no approval is given for a PR within a duration of 4 weeks after declaring it "ready", a PR can also be merged without other approvals + * Before doing so, the reasons for not approving must be found out (e.g. via the Chat / community call) + * If the reason simply is "no time" and there are no objections against that PR, the PR can be merged without other approvals From 114a35dc637f5ecea0cbac9c23317bdd92f1bb5d Mon Sep 17 00:00:00 2001 From: Andrey Balarev Date: Tue, 14 Feb 2023 15:13:11 +0200 Subject: [PATCH 042/173] Fixed from review issues in documentation. Signed-off-by: Andrey Balarev --- .../pages/ditto/connectivity-protocol-bindings-hono.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-hono.md b/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-hono.md index 940d6a8c0be..761e016ede3 100644 --- a/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-hono.md +++ b/documentation/src/main/resources/pages/ditto/connectivity-protocol-bindings-hono.md @@ -41,15 +41,15 @@ Runtime, these are resolved as following: * `event` -> `{%raw%}hono.event.{{connection:id}}{%endraw%}` * `telemetry` -> `{%raw%}hono.telemetry.{{connection:id}}{%endraw%}` * `command_response` -> `{%raw%}hono.command_response.{{connection:id}}{%endraw%}` - -Note: The {{connection:id}} will be replaced by the value of connectionId + +Note: The `{%raw%}{{connection:id}}{%endraw%}` will be replaced by the value of connectionId #### Source reply target Similar to source addresses, the reply target `address` is an alias as well. The single valid value for it is `command`. It is resolved to Kafka topic/key like this: * `command` -> `{%raw%}hono.command.{{connection:id}}/{%endraw%}` (<thingId> is substituted by thing ID value). -Note: The {{connection:id}} will be replaced by the value of connectionId +Note: The `{%raw%}{{connection:id}}{%endraw%}` will be replaced by the value of connectionId The needed header mappings for the `replyTarget` are also populated automatically at runtime and there is no need to specify them in the connection definition. Any of the following specified value will be substituted (i.e. ignored). @@ -101,7 +101,9 @@ and [Header mapping for connections](connectivity-header-mapping.html). #### Target address The target `address` is specified as an alias and the only valid alias is `command`. It is automatically resolved at runtime to the following Kafka topic/key: -* `command` -> `hono.command/` (<thingId> is substituted by thing ID value). +* `command` -> `{%raw%}hono.command.{{connection:id}}/{%endraw%}` (<thingId> is substituted by thing ID value). + +Note: The `{%raw%}{{connection:id}}{%endraw%}` will be replaced by the value of connectionId #### Target header mapping The target `headerMapping` section is also populated automatically at runtime and there is From c217920d072cda4edc73229d4d7c1d3f6ecfc008 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Tue, 14 Feb 2023 19:51:07 +0100 Subject: [PATCH 043/173] re-generated openapi docs Signed-off-by: Thomas Jaeckle --- documentation/src/main/resources/openapi/ditto-api-2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/src/main/resources/openapi/ditto-api-2.yml b/documentation/src/main/resources/openapi/ditto-api-2.yml index 18684504337..4f2fbe042d3 100644 --- a/documentation/src/main/resources/openapi/ditto-api-2.yml +++ b/documentation/src/main/resources/openapi/ditto-api-2.yml @@ -8017,6 +8017,8 @@ components: * ```like({property},{value})``` (i.e. contains values similar to the expressions listed) + * ```ilike({property},{value})``` (i.e. contains values similar and case insensitive to the expressions listed) + * ```exists({property})``` (i.e. all things in which the given path exists) From 2e2ecb13406ca0f4256d99fe014117f42184d921 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Wed, 15 Feb 2023 09:56:49 +0100 Subject: [PATCH 044/173] improve resiliency of connection id retrieval from journal by excluding empty events Signed-off-by: Thomas Jaeckle --- .../service/messaging/ConnectionIdsRetrievalActor.java | 8 ++++++++ .../ditto/internal/utils/persistentactors/EmptyEvent.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/ConnectionIdsRetrievalActor.java b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/ConnectionIdsRetrievalActor.java index 7e7e133c9a7..9373a78fb35 100644 --- a/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/ConnectionIdsRetrievalActor.java +++ b/connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/ConnectionIdsRetrievalActor.java @@ -40,6 +40,7 @@ import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory; import org.eclipse.ditto.internal.utils.cluster.DistPubSubAccess; import org.eclipse.ditto.internal.utils.persistence.mongo.streaming.MongoReadJournal; +import org.eclipse.ditto.internal.utils.persistentactors.EmptyEvent; import akka.NotUsed; import akka.actor.AbstractActor; @@ -114,6 +115,12 @@ private static boolean isDeleted(final Document document) { .orElse(true); } + private static boolean isNotEmptyEvent(final Document document) { + return Optional.ofNullable(document.getString(MongoReadJournal.J_EVENT_MANIFEST)) + .map(manifest -> !EmptyEvent.TYPE.equals(manifest)) + .orElse(false); + } + private static boolean isNotDeleted(final Document document) { return Optional.ofNullable(document.getString(MongoReadJournal.J_EVENT_MANIFEST)) .map(manifest -> !ConnectionDeleted.TYPE.equals(manifest)) @@ -166,6 +173,7 @@ private void getAllConnectionIDs(final WithDittoHeaders cmd) { final Source idsFromSnapshots = getIdsFromSnapshotsSource(); final Source idsFromJournal = persistenceIdsFromJournalSourceSupplier.get() .filter(ConnectionIdsRetrievalActor::isNotDeleted) + .filter(ConnectionIdsRetrievalActor::isNotEmptyEvent) .map(document -> document.getString(MongoReadJournal.J_EVENT_PID)); final CompletionStage retrieveAllConnectionIdsResponse = diff --git a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/EmptyEvent.java b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/EmptyEvent.java index 4d4d6c1df13..51d6a0430e0 100644 --- a/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/EmptyEvent.java +++ b/internal/utils/persistent-actors/src/main/java/org/eclipse/ditto/internal/utils/persistentactors/EmptyEvent.java @@ -56,7 +56,7 @@ public final class EmptyEvent implements Event { static final String NAME = "empty-event"; - static final String TYPE = TYPE_PREFIX + NAME; + public static final String TYPE = TYPE_PREFIX + NAME; private static final JsonFieldDefinition JSON_EFFECT = JsonFactory.newJsonValueFieldDefinition("effect", FieldType.REGULAR, JsonSchemaVersion.V_2); From 528edb9d0cd6e8e9d70d1047ce0a67283599a7a8 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Wed, 15 Feb 2023 10:53:45 +0100 Subject: [PATCH 045/173] added "ilike" to thingssearch-model Signed-off-by: Thomas Jaeckle --- .../ditto/thingsearch/model/ImmutableSearchProperty.java | 7 +++++++ .../eclipse/ditto/thingsearch/model/SearchFilter.java | 5 +++++ .../eclipse/ditto/thingsearch/model/SearchProperty.java | 9 +++++++++ .../thingsearch/model/ImmutableSearchPropertyTest.java | 8 ++++++++ 4 files changed, 29 insertions(+) diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchProperty.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchProperty.java index 166c9ceb140..123394aed9f 100755 --- a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchProperty.java +++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchProperty.java @@ -240,6 +240,13 @@ public PropertySearchFilter like(final String value) { return ImmutablePropertyFilter.of(SearchFilter.Type.LIKE, propertyPath, JsonFactory.newValue(value)); } + @Override + public PropertySearchFilter ilike(final String value) { + checkStringValue(value); + + return ImmutablePropertyFilter.of(SearchFilter.Type.ILIKE, propertyPath, JsonFactory.newValue(value)); + } + @Override public PropertySearchFilter in(final boolean value, final Boolean... furtherValues) { return in(toCollection(JsonFactory::newValue, value, furtherValues)); diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchFilter.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchFilter.java index 57d47a3c347..d56e192ca9c 100755 --- a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchFilter.java +++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchFilter.java @@ -99,6 +99,11 @@ enum Type { */ LIKE("like"), + /** + * Filter type for checking if a string matches a regular expression, in a case insensitive way. + */ + ILIKE("ilike"), + /** * Filter type for checking if an entity is contained in a set of given entities. */ diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchProperty.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchProperty.java index 4a2850868c2..5f9dccd95f3 100755 --- a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchProperty.java +++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/SearchProperty.java @@ -293,6 +293,15 @@ public interface SearchProperty { */ PropertySearchFilter like(String value); + /** + * Returns a new search filter for checking if the value of this property is case insensitive like the given value. + * + * @param value the value to compare the value of this property with. + * @return the new search filter. + * @throws NullPointerException if {@code value} is {@code null}. + */ + PropertySearchFilter ilike(String value); + /** * Returns a new search filter for checking if the value of this property is in the given value(s). * diff --git a/thingsearch/model/src/test/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchPropertyTest.java b/thingsearch/model/src/test/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchPropertyTest.java index 3e1b07c90dc..0ce96aa648e 100755 --- a/thingsearch/model/src/test/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchPropertyTest.java +++ b/thingsearch/model/src/test/java/org/eclipse/ditto/thingsearch/model/ImmutableSearchPropertyTest.java @@ -207,6 +207,14 @@ public void likeReturnsExpected() { .hasStringRepresentation("like(" + PROPERTY_PATH + ",\"" + BOSCH + "\")"); } + @Test + public void ilikeReturnsExpected() { + assertThat(underTest.ilike(BOSCH)) + .hasType(SearchFilter.Type.ILIKE) + .hasOnlyValue(BOSCH) + .hasStringRepresentation("ilike(" + PROPERTY_PATH + ",\"" + BOSCH + "\")"); + } + @Test(expected = NullPointerException.class) public void tryToCallInWithNullStringForMandatoryValue() { underTest.in(null, ACME); From ef97a0cca27ef0377b2aaebf44aa92cca1570c44 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Wed, 15 Feb 2023 11:39:06 +0100 Subject: [PATCH 046/173] ignore very unstable unit test Signed-off-by: Thomas Jaeckle --- .../messaging/persistence/ConnectionPersistenceActorTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java index 29e65232d02..5fc675a4de2 100644 --- a/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java +++ b/connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/persistence/ConnectionPersistenceActorTest.java @@ -94,6 +94,7 @@ import org.eclipse.ditto.thingsearch.model.signals.commands.subscription.CreateSubscription; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.mockito.Mockito; @@ -1241,6 +1242,7 @@ private static boolean isMessageSenderInstanceOf(final Object message, final Cla } @Test + @Ignore("TODO unignore and stabilize flaky test") public void retriesStartingClientActor() { final var parent = actorSystemResource1.newTestProbe(); final var underTest = parent.childActorOf( From dd79a8c81237eb857d3e9f72896c5dd801f799b0 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Mon, 20 Feb 2023 11:32:21 +0100 Subject: [PATCH 047/173] removed access token from star history chart --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ee6d28b749c..cc627a66e31 100755 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ You should be able to work with your locally running default using the `local_di ## Star History -[![Star History Chart](https://api.star-history.com/svg?repos=eclipse-ditto/ditto&type=Date)](https://star-history.com/?secret=Z2hwXzJERUNBUmFRa09KM3BvdTFMUkJ1Y3VnY25FV3hxVjNBM3hEVQ==#eclipse-ditto/ditto&Date) +[![Star History Chart](https://api.star-history.com/svg?repos=eclipse-ditto/ditto&type=Date)](https://star-history.com/#eclipse-ditto/ditto&Date) ## Getting started From 02ce409095b9ed39f7917198f05ba049b222f2b7 Mon Sep 17 00:00:00 2001 From: thfries Date: Mon, 13 Feb 2023 20:44:14 +0100 Subject: [PATCH 048/173] Explorer UI - autocomplete for search Signed-off-by: thfries --- ui/index.css | 8 ++ ui/index.html | 1 + ui/modules/connections/connections.html | 3 + ui/modules/connections/connectionsMonitor.js | 58 +++++++++- ui/modules/things/searchFilter.js | 110 ++++++++++++++++--- ui/modules/things/things.html | 4 +- ui/modules/utils.js | 47 +++++++- 7 files changed, 205 insertions(+), 26 deletions(-) diff --git a/ui/index.css b/ui/index.css index 401a2c5ab0c..0f725c5f5e5 100644 --- a/ui/index.css +++ b/ui/index.css @@ -179,4 +179,12 @@ hr { h5>.badge { vertical-align: top; font-size: 0.6em; +} + +.autoComplete_wrapper { + flex-grow: 1; +} + +.autoComplete_wrapper ul > li[aria-selected="true"] { + background-color: rgba(123, 123, 123, 0.1); } \ No newline at end of file diff --git a/ui/index.html b/ui/index.html index d9d9935f995..6905061c8a3 100644 --- a/ui/index.html +++ b/ui/index.html @@ -45,6 +45,7 @@ integrity="sha512-GZ1RIgZaSc8rnco/8CXfRdCpDxRCphenIiZ2ztLy3XQfCbQUSCuk8IudvNHxkRA3oUg6q0qejgN/qqyG1duv5Q==" crossorigin="anonymous" type="text/javascript" charset="utf-8"> + diff --git a/ui/modules/connections/connections.html b/ui/modules/connections/connections.html index a191afa1ff4..f026ff9ef87 100644 --- a/ui/modules/connections/connections.html +++ b/ui/modules/connections/connections.html @@ -166,6 +166,9 @@
    Connection Logs
    Refresh +
    diff --git a/ui/modules/connections/connectionsMonitor.js b/ui/modules/connections/connectionsMonitor.js index 1d313c389f2..3695803f422 100644 --- a/ui/modules/connections/connectionsMonitor.js +++ b/ui/modules/connections/connectionsMonitor.js @@ -29,6 +29,7 @@ let dom = { buttonRetrieveConnectionMetrics: null, buttonResetConnectionMetrics: null, tableValidationConnections: null, + // inputConnectionLogFilter: null, }; let connectionLogs; @@ -60,6 +61,7 @@ export function ready() { dom.buttonRetrieveConnectionMetrics.onclick = retrieveConnectionMetrics; document.querySelector('a[data-bs-target="#tabConnectionMetrics"]').onclick = retrieveConnectionMetrics; dom.buttonResetConnectionMetrics.onclick = onResetConnectionMetricsClick; + // dom.inputConnectionLogFilter.onchange = onConnectionLogFilterChange; } function onResetConnectionMetricsClick() { @@ -112,19 +114,27 @@ function retrieveConnectionStatus() { function retrieveConnectionLogs() { Utils.assert(selectedConnectionId, 'Please select a connection', dom.tableValidationConnections); - dom.tbodyConnectionLogs.innerHTML = ''; - connectionLogDetail.setValue(''); API.callConnectionsAPI('retrieveConnectionLogs', (response) => { connectionLogs = response.connectionLogs; adjustEnableButton(response); - response.connectionLogs.forEach((entry) => { - Utils.addTableRow(dom.tbodyConnectionLogs, Utils.formatDate(entry.timestamp, true), false, false, entry.type, entry.level); - }); - dom.tbodyConnectionLogs.scrollTop = dom.tbodyConnectionLogs.scrollHeight - dom.tbodyConnectionLogs.clientHeight; + fillConnectionLogsTable(response.connectionLogs); }, selectedConnectionId); } +let connectionLogsFilter; + +function fillConnectionLogsTable(entries) { + dom.tbodyConnectionLogs.innerHTML = ''; + connectionLogDetail.setValue(''); + + let filter = connectionLogsFilter ? connectionLogsFilter.match : (a => true); + entries.filter(filter).forEach((entry) => { + Utils.addTableRow(dom.tbodyConnectionLogs, Utils.formatDate(entry.timestamp, true), false, false, entry.type, entry.level); + }); + dom.tbodyConnectionLogs.scrollTop = dom.tbodyConnectionLogs.scrollHeight - dom.tbodyConnectionLogs.clientHeight; +} + function adjustEnableButton(response) { if (response.enabledUntil) { dom.buttonEnableConnectionLogs.querySelector('i').classList.replace('bi-toggle-off', 'bi-toggle-on'); @@ -145,3 +155,39 @@ function onConnectionChange(connection, isNewConnection = true) { retrieveConnectionLogs(); } } + +function JsonFilter() { + let _filters = []; + + const match = (object) => { + let result = true; + _filters.forEach((f) => result = result && object[f.key] === f.value); + return result; + }; + + const add = (key, value) => { + _filters.push({key: key, value: value}); + }; + + return { + match, + add, + }; +} + +const knownFields = ['category', 'type', 'level']; + +function onConnectionLogFilterChange(event) { + if (event.target.value && event.target.value !== '') { + connectionLogsFilter = new JsonFilter(); + event.target.value.split(/(\s+)/).forEach((elem) => { + const keyValue = elem.split(':'); + if (keyValue.length === 2 && knownFields.includes(keyValue[0].trim())) { + connectionLogsFilter.add(keyValue[0].trim(), keyValue[1].trim()); + } + }); + } else { + connectionLogsFilter = null; + } + fillConnectionLogsTable(connectionLogs); +} diff --git a/ui/modules/things/searchFilter.js b/ui/modules/things/searchFilter.js index fbbbe2f72ef..077b4449e1f 100644 --- a/ui/modules/things/searchFilter.js +++ b/ui/modules/things/searchFilter.js @@ -31,7 +31,9 @@ const filterHistory = []; let keyStrokeTimeout; -const FILTER_PLACEHOLDER = '*****'; +const FILTER_PLACEHOLDER = '...'; + +let autoCompleteJS; const dom = { filterList: null, @@ -53,15 +55,15 @@ export async function ready() { dom.pinnedThings.onclick = ThingsSearch.pinnedTriggered; + autoCompleteJS = Utils.createAutoComplete('#searchFilterEdit', createFilterList, 'Search for Things...'); + autoCompleteJS.input.addEventListener('selection', (event) => { + const selection = event.detail.selection.value; + fillSearchFilterEdit(selection.rql); + }); dom.filterList.addEventListener('click', (event) => { if (event.target && event.target.classList.contains('dropdown-item')) { - dom.searchFilterEdit.value = event.target.textContent; - checkIfFavourite(); - const filterEditNeeded = checkAndMarkParameter(); - if (!filterEditNeeded) { - ThingsSearch.searchTriggered(event.target.textContent); - } + fillSearchFilterEdit(event.target.textContent); } }); @@ -78,7 +80,7 @@ export async function ready() { }; dom.searchFilterEdit.onkeyup = (event) => { - if (event.key === 'Enter' || event.code === 13) { + if ((event.key === 'Enter' || event.code === 13) && dom.searchFilterEdit.value.indexOf(FILTER_PLACEHOLDER) < 0) { fillHistory(dom.searchFilterEdit.value); ThingsSearch.searchTriggered(dom.searchFilterEdit.value); } else { @@ -87,14 +89,10 @@ export async function ready() { } }; - dom.searchFilterEdit.onclick = (event) => { - if (event.target.selectionStart === event.target.selectionEnd) { - event.target.select(); - } + dom.searchFilterEdit.onchange = () => { + ThingsSearch.removeMoreFromThingList(); }; - dom.searchFilterEdit.onchange = ThingsSearch.removeMoreFromThingList; - dom.searchFilterEdit.focus(); } @@ -111,6 +109,16 @@ function onEnvironmentChanged() { updateFilterList(); } +function fillSearchFilterEdit(fillString) { + dom.searchFilterEdit.value = fillString; + + checkIfFavourite(); + const filterEditNeeded = checkAndMarkParameter(); + if (!filterEditNeeded) { + ThingsSearch.searchTriggered(dom.searchFilterEdit.value); + } +} + /** * Updates the UI filterList */ @@ -118,15 +126,81 @@ function updateFilterList() { dom.filterList.innerHTML = ''; Utils.addDropDownEntries(dom.filterList, ['Favourite search filters'], true); Utils.addDropDownEntries(dom.filterList, Environments.current().filterList ?? []); - Utils.addDropDownEntries(dom.filterList, ['Field search filters'], true); - Utils.addDropDownEntries(dom.filterList, (Environments.current().fieldList ?? []) - .map((f) => `eq(${f.path},${FILTER_PLACEHOLDER})`)); Utils.addDropDownEntries(dom.filterList, ['Example search filters'], true); Utils.addDropDownEntries(dom.filterList, filterExamples); Utils.addDropDownEntries(dom.filterList, ['Recent search filters'], true); Utils.addDropDownEntries(dom.filterList, filterHistory); } +async function createFilterList(query) { + const date24h = new Date(); + const date1h = new Date(); + const date1m = new Date(); + date24h.setDate(date24h.getDate() - 1); + date1h.setHours(date1h.getHours() - 1); + date1m.setMinutes(date1m.getMinutes() -1); + + return [ + { + label: 'Created since 1m', + rql: `gt(_created,"${date1m.toISOString()}")`, + group: 'Time', + }, + { + label: 'Created since 1h', + rql: `gt(_created,"${date1h.toISOString()}")`, + group: 'Time', + }, + { + label: 'Created since 24h', + rql: `gt(_created,"${date24h.toISOString()}")`, + group: 'Time', + }, + { + label: 'Modified since 1m', + rql: `gt(_modified,"${date1m.toISOString()}")`, + group: 'Time', + }, + { + label: 'Modified since 1h', + rql: `gt(_modified,"${date1h.toISOString()}")`, + group: 'Time', + }, + { + label: 'Modified since 24h', + rql: `gt(_modified,"${date24h.toISOString()}")`, + group: 'Time', + }, + { + label: `thingId = ${FILTER_PLACEHOLDER}`, + rql: `eq(thingId,"${FILTER_PLACEHOLDER}")`, + group: 'ThingId', + }, + { + label: `thingId ~ ${FILTER_PLACEHOLDER}`, + rql: `like(thingId,"${FILTER_PLACEHOLDER}*")`, + group: 'ThingId', + }, + { + label: `exists ${FILTER_PLACEHOLDER}`, + rql: `exists(${FILTER_PLACEHOLDER})`, + group: 'Other', + }, + ...(Environments.current().filterList ?? []).map((f) => ({label: f, rql: f, group: 'Favourite'})), + ...(Environments.current().fieldList ?? []).map((f) => ({ + label: `${f.label} = ${FILTER_PLACEHOLDER}`, + rql: `eq(${f.path},"${FILTER_PLACEHOLDER}")`, + group: 'Field', + })), + ...(Environments.current().fieldList ?? []).map((f) => ({ + label: `${f.label} ~ ${FILTER_PLACEHOLDER}`, + rql: `like(${f.path},"*${FILTER_PLACEHOLDER}*")`, + group: 'Field', + })), + ...filterHistory.map((f) => ({label: f, rql: f, group: 'Recent'})), + ]; +} + /** * Adds or removes the given filter from the list of search filters * @param {String} filter filter @@ -160,6 +234,8 @@ function checkIfFavourite() { function checkAndMarkParameter() { const index = dom.searchFilterEdit.value.indexOf(FILTER_PLACEHOLDER); if (index >= 0) { + // filterString.replace(FILTER_PLACEHOLDER, ''); + // dom.searchFilterEdit.value = filterString; dom.searchFilterEdit.focus(); dom.searchFilterEdit.setSelectionRange(index, index + FILTER_PLACEHOLDER.length); return true; diff --git a/ui/modules/things/things.html b/ui/modules/things/things.html index 718866f7761..272c137281a 100644 --- a/ui/modules/things/things.html +++ b/ui/modules/things/things.html @@ -26,8 +26,8 @@
    Things
    data-bs-toggle="dropdown"> - +
    modifyTargetActorCommandResponse(final Signal< pair.response() instanceof RetrieveThingResponse retrieveThingResponse) { return inlinePolicyEnrichment.enrichPolicy(retrieveThing, retrieveThingResponse) .map(Object.class::cast); + } else if (RollbackCreatedPolicy.shouldRollback(pair.command(), pair.response())) { + CompletableFuture responseF = new CompletableFuture<>(); + getSelf().tell(RollbackCreatedPolicy.of(pair.command(), pair.response(), responseF), getSelf()); + return Source.fromCompletionStage(responseF); } else { return Source.single(pair.response()); } @@ -315,6 +329,36 @@ protected CompletionStage modifyTargetActorCommandResponse(final Signal< .run(materializer); } + @Override + protected CompletableFuture handleTargetActorException(final Throwable error, final Signal enforcedSignal) { + if (enforcedSignal instanceof CreateThing createThing) { + CompletableFuture responseFuture = new CompletableFuture<>(); + getSelf().tell(RollbackCreatedPolicy.of(createThing, error, responseFuture), getSelf()); + return responseFuture; + } + return CompletableFuture.failedFuture(error); + } + + private void handlerRollbackCreatedPolicy(final RollbackCreatedPolicy rollback) { + if (policyCreatedEvent != null) { + DittoHeaders dittoHeaders = rollback.initialCommand().getDittoHeaders(); + final DeletePolicy deletePolicy = DeletePolicy.of(policyCreatedEvent.policyId(), dittoHeaders.toBuilder() + .putHeader(DittoHeaderDefinition.DITTO_SUDO.getKey(), "true").build()); + AskWithRetry.askWithRetry(policiesShardRegion, deletePolicy, + enforcementConfig.getAskWithRetryConfig(), + getContext().system(), response -> { + log.withCorrelationId(dittoHeaders) + .info("Policy <{}> deleted after rolling back it's creation. " + + "Policies shard region response: <{}>", deletePolicy.getEntityId(), response); + rollback.completeInitialResponse(); + return response; + }); + + } else { + rollback.completeInitialResponse(); + } + } + @Override protected ThingId getEntityId() throws Exception { return ThingId.of(URLDecoder.decode(getSelf().path().name(), StandardCharsets.UTF_8)); @@ -377,6 +421,10 @@ protected Receive activeBehaviour(final Runnable matchProcessNextTwinMessageBeha final FI.UnitApply matchAnyBehavior) { return ReceiveBuilder.create() .matchEquals(Control.SHUTDOWN_TIMEOUT, this::shutdownActor) + .match(ThingPolicyCreated.class, event -> { + this.policyCreatedEvent = event; + log.withCorrelationId(event.dittoHeaders()).info("Policy <{}> created", event.policyId()); + }).match(RollbackCreatedPolicy.class, this::handlerRollbackCreatedPolicy) .build() .orElse(super.activeBehaviour(matchProcessNextTwinMessageBehavior, matchAnyBehavior)); } From 10243357966e1d6d8ed8b0ad57c262d916f6c36f Mon Sep 17 00:00:00 2001 From: thfries Date: Thu, 23 Feb 2023 19:44:42 +0100 Subject: [PATCH 051/173] Explorer UI - change favorite spelling Signed-off-by: thfries --- ui/modules/things/featureMessages.html | 4 ++-- ui/modules/things/featureMessages.js | 4 ++-- ui/modules/things/searchFilter.js | 18 +++++++++--------- ui/modules/things/things.html | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ui/modules/things/featureMessages.html b/ui/modules/things/featureMessages.html index 1fee8384e90..c35f30fd50b 100644 --- a/ui/modules/things/featureMessages.html +++ b/ui/modules/things/featureMessages.html @@ -25,8 +25,8 @@
    - - -
    diff --git a/ui/modules/utils.js b/ui/modules/utils.js index 22520be2720..6b69fff8eb5 100644 --- a/ui/modules/utils.js +++ b/ui/modules/utils.js @@ -321,7 +321,7 @@ export function createAutoComplete(selector, src, placeHolder) { selector: selector, data: { src: src, - keys: ['label'], + keys: ['label', 'group'], }, placeHolder: placeHolder, resultsList: { @@ -333,8 +333,9 @@ export function createAutoComplete(selector, src, placeHolder) { highlight: true, element: (item, data) => { item.style = 'display: flex;'; - item.innerHTML = `${data.match} - ${data.value.group}`; + item.innerHTML = `${data.key === 'label' ? data.match : data.value.label} + + ${data.key === 'group' ? data.match : data.value.group}`; }, }, events: { From 03aa5a5255a0914ac1fa6ef67effbc5dea09c372 Mon Sep 17 00:00:00 2001 From: Bob Claerhout Date: Tue, 28 Feb 2023 09:00:42 +0100 Subject: [PATCH 053/173] fix href in docs Signed-off-by: Bob Claerhout --- .../src/main/resources/pages/ditto/installation-operating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/src/main/resources/pages/ditto/installation-operating.md b/documentation/src/main/resources/pages/ditto/installation-operating.md index fb992b7ce28..e12756f5d16 100644 --- a/documentation/src/main/resources/pages/ditto/installation-operating.md +++ b/documentation/src/main/resources/pages/ditto/installation-operating.md @@ -37,7 +37,7 @@ In order to have a look at all possible configuration options and what default v configuration files of Ditto's microservices: * Policies: [policies.conf](https://github.com/eclipse-ditto/ditto/blob/master/policies/service/src/main/resources/policies.conf) * Things: [things.conf](https://github.com/eclipse-ditto/ditto/blob/master/things/service/src/main/resources/things.conf) -* Things-Search: [things-search.conf](https://github.com/eclipse-ditto/ditto/blob/master/thingsearch/service/src/main/resources/things-search.conf) +* Things-Search: [things-search.conf](https://github.com/eclipse-ditto/ditto/blob/master/thingsearch/service/src/main/resources/search.conf) * Connectivity: [connectivity.conf](https://github.com/eclipse-ditto/ditto/blob/master/connectivity/service/src/main/resources/connectivity.conf) * Gateway: [gateway.conf](https://github.com/eclipse-ditto/ditto/blob/master/gateway/service/src/main/resources/gateway.conf) From c13213136d064e487874479f11d3aacdc40ec08e Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Mon, 6 Mar 2023 20:12:58 +0100 Subject: [PATCH 054/173] [#1592] provide "Bearer" authentication for devops resources controlled via UI * added radiobutton groups so that "Authorize" modal clearly shows/states which authentication to use where * reduced to a single "Authorize" button in modal * simplified environment a bit Signed-off-by: Thomas Jaeckle --- ui/index.css | 4 + ui/modules/api.js | 29 +++-- ui/modules/environments/authorization.html | 126 ++++++++++++--------- ui/modules/environments/authorization.js | 56 ++++++--- ui/templates/environmentTemplates.json | 23 ++-- 5 files changed, 148 insertions(+), 90 deletions(-) diff --git a/ui/index.css b/ui/index.css index 0f725c5f5e5..347968dbfe0 100644 --- a/ui/index.css +++ b/ui/index.css @@ -46,6 +46,10 @@ body { padding-top: 60px; } +#authorizationModal label { + display: inline; +} + .toast-header { color: #842029; background-color: #f8d7da; diff --git a/ui/modules/api.js b/ui/modules/api.js index cdadd62a39e..eb343316f2b 100644 --- a/ui/modules/api.js +++ b/ui/modules/api.js @@ -277,18 +277,25 @@ let authHeaderValue; * @param {boolean} forDevOps if true, the credentials for the dev ops api will be used. */ export function setAuthHeader(forDevOps) { - if (forDevOps && Environments.current().useBasicAuth) { - authHeaderKey = 'Authorization'; - authHeaderValue = 'Basic ' + window.btoa(Environments.current().usernamePasswordDevOps); - } else if (Environments.current().useBasicAuth) { - authHeaderKey = 'Authorization'; - authHeaderValue = 'Basic ' + window.btoa(Environments.current().usernamePassword); - } else if (Environments.current().useDittoPreAuthenticatedAuth) { - authHeaderKey = 'x-ditto-pre-authenticated'; - authHeaderValue = Environments.current().dittoPreAuthenticatedUsername; + if (forDevOps) { + if (Environments.current().devopsAuth === 'basic') { + authHeaderKey = 'Authorization'; + authHeaderValue = 'Basic ' + window.btoa(Environments.current().usernamePasswordDevOps); + } else if (Environments.current().devopsAuth === 'bearer') { + authHeaderKey = 'Authorization'; + authHeaderValue ='Bearer ' + Environments.current().bearerDevOps; + } } else { - authHeaderKey = 'Authorization'; - authHeaderValue ='Bearer ' + Environments.current().bearer; + if (Environments.current().mainAuth === 'basic') { + authHeaderKey = 'Authorization'; + authHeaderValue = 'Basic ' + window.btoa(Environments.current().usernamePassword); + } else if (Environments.current().mainAuth === 'pre') { + authHeaderKey = 'x-ditto-pre-authenticated'; + authHeaderValue = Environments.current().dittoPreAuthenticatedUsername; + } else if (Environments.current().mainAuth === 'bearer') { + authHeaderKey = 'Authorization'; + authHeaderValue ='Bearer ' + Environments.current().bearer; + } } } diff --git a/ui/modules/environments/authorization.html b/ui/modules/environments/authorization.html index 91637f2fe04..ef3a0b88486 100644 --- a/ui/modules/environments/authorization.html +++ b/ui/modules/environments/authorization.html @@ -18,78 +18,94 @@