From 84d51074fdfb4c4387ba1827a9cd75555cf2cc59 Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger <43503240+paullatzelsperger@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:38:38 +0200 Subject: [PATCH] feat: use the SqlSchemaBootstrapper to prep DBs (#346) feat: use the SqlSchemaBootstrapper to prepare the databases --- deployment/assets/postgres/edc_schema.sql | 301 ------------------ deployment/assets/postgres/ih_schema.sql | 75 ----- deployment/consumer.tf | 8 +- .../modules/catalog-server/catalog-server.tf | 1 + deployment/modules/connector/controlplane.tf | 1 + deployment/modules/connector/dataplane.tf | 1 + deployment/modules/identity-hub/main.tf | 1 + deployment/provider.tf | 20 +- 8 files changed, 10 insertions(+), 398 deletions(-) delete mode 100644 deployment/assets/postgres/edc_schema.sql delete mode 100644 deployment/assets/postgres/ih_schema.sql diff --git a/deployment/assets/postgres/edc_schema.sql b/deployment/assets/postgres/edc_schema.sql deleted file mode 100644 index 6734fddb..00000000 --- a/deployment/assets/postgres/edc_schema.sql +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Copyright (c) 2024 Metaform Systems, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Metaform Systems, Inc. - initial API and implementation - * - */ - --- table: edc_asset -CREATE TABLE IF NOT EXISTS edc_asset -( - asset_id VARCHAR NOT NULL, - created_at BIGINT NOT NULL, - properties JSON DEFAULT '{}', - private_properties JSON DEFAULT '{}', - data_address JSON DEFAULT '{}', - PRIMARY KEY (asset_id) - ); - -COMMENT ON COLUMN edc_asset.properties IS 'Asset properties serialized as JSON'; -COMMENT ON COLUMN edc_asset.private_properties IS 'Asset private properties serialized as JSON'; -COMMENT ON COLUMN edc_asset.data_address IS 'Asset DataAddress serialized as JSON'; - --- table: edc_contract_definitions --- only intended for and tested with H2 and Postgres! -CREATE TABLE IF NOT EXISTS edc_contract_definitions -( - created_at BIGINT NOT NULL, - contract_definition_id VARCHAR NOT NULL, - access_policy_id VARCHAR NOT NULL, - contract_policy_id VARCHAR NOT NULL, - assets_selector JSON NOT NULL, - private_properties JSON, - PRIMARY KEY (contract_definition_id) -); - - -CREATE TABLE IF NOT EXISTS edc_lease -( - leased_by VARCHAR NOT NULL, - leased_at BIGINT, - lease_duration INTEGER DEFAULT 60000 NOT NULL, - lease_id VARCHAR NOT NULL - CONSTRAINT lease_pk - PRIMARY KEY -); - -COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; - -COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; - - -CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex - ON edc_lease (lease_id); - - - -CREATE TABLE IF NOT EXISTS edc_contract_agreement -( - agr_id VARCHAR NOT NULL - CONSTRAINT contract_agreement_pk - PRIMARY KEY, - provider_agent_id VARCHAR, - consumer_agent_id VARCHAR, - signing_date BIGINT, - start_date BIGINT, - end_date INTEGER, - asset_id VARCHAR NOT NULL, - policy JSON -); - - -CREATE TABLE IF NOT EXISTS edc_contract_negotiation -( - id VARCHAR NOT NULL - CONSTRAINT contract_negotiation_pk - PRIMARY KEY, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL, - correlation_id VARCHAR, - counterparty_id VARCHAR NOT NULL, - counterparty_address VARCHAR NOT NULL, - protocol VARCHAR NOT NULL, - type VARCHAR NOT NULL, - state INTEGER DEFAULT 0 NOT NULL, - state_count INTEGER DEFAULT 0, - state_timestamp BIGINT, - error_detail VARCHAR, - agreement_id VARCHAR - CONSTRAINT contract_negotiation_contract_agreement_id_fk - REFERENCES edc_contract_agreement, - contract_offers JSON, - callback_addresses JSON, - trace_context JSON, - pending BOOLEAN DEFAULT FALSE, - protocol_messages JSON, - lease_id VARCHAR - CONSTRAINT contract_negotiation_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL -); - -COMMENT ON COLUMN edc_contract_negotiation.agreement_id IS 'ContractAgreement serialized as JSON'; - -COMMENT ON COLUMN edc_contract_negotiation.contract_offers IS 'List serialized as JSON'; - -COMMENT ON COLUMN edc_contract_negotiation.trace_context IS 'Map serialized as JSON'; - - -CREATE INDEX IF NOT EXISTS contract_negotiation_correlationid_index - ON edc_contract_negotiation (correlation_id); - -CREATE UNIQUE INDEX IF NOT EXISTS contract_negotiation_id_uindex - ON edc_contract_negotiation (id); - -CREATE UNIQUE INDEX IF NOT EXISTS contract_agreement_id_uindex - ON edc_contract_agreement (agr_id); - - --- table: edc_policydefinitions -CREATE TABLE IF NOT EXISTS edc_policydefinitions -( - policy_id VARCHAR NOT NULL, - created_at BIGINT NOT NULL, - permissions JSON, - prohibitions JSON, - duties JSON, - profiles JSON, - extensible_properties JSON, - inherits_from VARCHAR, - assigner VARCHAR, - assignee VARCHAR, - target VARCHAR, - policy_type VARCHAR NOT NULL, - private_properties JSON, - PRIMARY KEY (policy_id) -); - -COMMENT ON COLUMN edc_policydefinitions.permissions IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.prohibitions IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.duties IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.profiles IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.extensible_properties IS 'Java Map serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.policy_type IS 'Java PolicyType serialized as JSON'; - -CREATE UNIQUE INDEX IF NOT EXISTS edc_policydefinitions_id_uindex - ON edc_policydefinitions (policy_id); - - - -CREATE TABLE IF NOT EXISTS edc_transfer_process -( - transferprocess_id VARCHAR NOT NULL - CONSTRAINT transfer_process_pk - PRIMARY KEY, - type VARCHAR NOT NULL, - state INTEGER NOT NULL, - state_count INTEGER DEFAULT 0 NOT NULL, - state_time_stamp BIGINT, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL, - trace_context JSON, - error_detail VARCHAR, - resource_manifest JSON, - provisioned_resource_set JSON, - content_data_address JSON, - deprovisioned_resources JSON, - private_properties JSON, - callback_addresses JSON, - pending BOOLEAN DEFAULT FALSE, - transfer_type VARCHAR, - protocol_messages JSON, - data_plane_id VARCHAR, - correlation_id VARCHAR, - counter_party_address VARCHAR, - protocol VARCHAR, - asset_id VARCHAR, - contract_id VARCHAR, - data_destination JSON, - lease_id VARCHAR - CONSTRAINT transfer_process_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL -); - -COMMENT ON COLUMN edc_transfer_process.trace_context IS 'Java Map serialized as JSON'; - - -COMMENT ON COLUMN edc_transfer_process.resource_manifest IS 'java ResourceManifest serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.provisioned_resource_set IS 'ProvisionedResourceSet serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.content_data_address IS 'DataAddress serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.deprovisioned_resources IS 'List of deprovisioned resources, serialized as JSON'; - - -CREATE UNIQUE INDEX IF NOT EXISTS transfer_process_id_uindex - ON edc_transfer_process (transferprocess_id); - - -CREATE TABLE IF NOT EXISTS edc_data_plane_instance -( - id VARCHAR NOT NULL PRIMARY KEY, - data JSON, - lease_id VARCHAR - CONSTRAINT data_plane_instance_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL -); - - -CREATE TABLE IF NOT EXISTS edc_policy_monitor -( - entry_id VARCHAR NOT NULL PRIMARY KEY, - state INTEGER NOT NULL , - created_at BIGINT NOT NULL , - updated_at BIGINT NOT NULL , - state_count INTEGER DEFAULT 0 NOT NULL, - state_time_stamp BIGINT, - trace_context JSON, - error_detail VARCHAR, - lease_id VARCHAR - CONSTRAINT policy_monitor_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL, - properties JSON, - contract_id VARCHAR -); - - -CREATE TABLE IF NOT EXISTS edc_edr_entry -( - transfer_process_id VARCHAR NOT NULL PRIMARY KEY, - agreement_id VARCHAR NOT NULL, - asset_id VARCHAR NOT NULL, - provider_id VARCHAR NOT NULL, - contract_negotiation_id VARCHAR, - created_at BIGINT NOT NULL -); - --- ################## --- DATA PLANE TABLES --- ################## - -CREATE TABLE IF NOT EXISTS edc_accesstokendata -( - id VARCHAR NOT NULL PRIMARY KEY, - claim_token JSON NOT NULL, - data_address JSON NOT NULL, - additional_properties JSON DEFAULT '{}' -); - -CREATE TABLE IF NOT EXISTS edc_lease -( - leased_by VARCHAR NOT NULL, - leased_at BIGINT, - lease_duration INTEGER NOT NULL, - lease_id VARCHAR NOT NULL - CONSTRAINT lease_pk - PRIMARY KEY -); - - -CREATE TABLE IF NOT EXISTS edc_data_plane -( - process_id VARCHAR NOT NULL PRIMARY KEY, - state INTEGER NOT NULL , - created_at BIGINT NOT NULL , - updated_at BIGINT NOT NULL , - state_count INTEGER DEFAULT 0 NOT NULL, - state_time_stamp BIGINT, - trace_context JSON, - error_detail VARCHAR, - callback_address VARCHAR, - lease_id VARCHAR - CONSTRAINT data_plane_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL, - source JSON, - destination JSON, - properties JSON, - flow_type VARCHAR, - transfer_type_destination VARCHAR -); - --- This will help to identify states that need to be transitioned without a table scan when the entries grow -CREATE INDEX IF NOT EXISTS data_plane_state ON edc_data_plane (state,state_time_stamp); - -CREATE TABLE IF NOT EXISTS edc_federated_catalog -( - id VARCHAR PRIMARY KEY NOT NULL, - catalog JSON, - marked BOOLEAN DEFAULT FALSE -); diff --git a/deployment/assets/postgres/ih_schema.sql b/deployment/assets/postgres/ih_schema.sql deleted file mode 100644 index 635a1efb..00000000 --- a/deployment/assets/postgres/ih_schema.sql +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2024 Metaform Systems, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Metaform Systems, Inc. - initial API and implementation - * - */ - - - -CREATE TABLE IF NOT EXISTS credential_resource -( - id VARCHAR PRIMARY KEY NOT NULL, -- ID of the VC, duplicated here for indexing purposes - create_timestamp BIGINT NOT NULL, -- POSIX timestamp of the creation of the VC - issuer_id VARCHAR NOT NULL, - holder_id VARCHAR NOT NULL, - vc_state INTEGER NOT NULL, - issuance_policy JSON, - reissuance_policy JSON, - raw_vc VARCHAR NOT NULL, -- Representation of the VC exactly as it was received by the issuer. Can be JWT or JSON(-LD) - vc_format INTEGER NOT NULL, -- 0 = JSON-LD, 1 = JWT - verifiable_credential JSON NOT NULL, -- JSON-representation of the verifiable credential - participant_id VARCHAR -- ID of the ParticipantContext that owns this credentisl -); -CREATE UNIQUE INDEX IF NOT EXISTS credential_resource_credential_id_uindex ON credential_resource USING btree (id); -COMMENT ON COLUMN credential_resource.id IS 'ID of the VC, duplicated here for indexing purposes'; -COMMENT ON COLUMN credential_resource.raw_vc IS 'Representation of the VC exactly as it was received by the issuer. Can be JWT or JSON(-LD) '; -COMMENT ON COLUMN credential_resource.vc_format IS '0 = JSON-LD, 1 = JWT'; -COMMENT ON COLUMN credential_resource.verifiable_credential IS 'JSON-representation of the VerifiableCredential'; - -CREATE TABLE IF NOT EXISTS did_resources -( - did VARCHAR NOT NULL, - create_timestamp BIGINT NOT NULL, - state_timestamp BIGINT NOT NULL, - state INT NOT NULL, - did_document JSON NOT NULL, - participant_id VARCHAR, - PRIMARY KEY (did) -); - -CREATE TABLE IF NOT EXISTS keypair_resource -( - id VARCHAR PRIMARY KEY NOT NULL, -- primary key - participant_id VARCHAR, -- ID of the owning ParticipantContext. this is a loose business key, not a FK! - timestamp BIGINT NOT NULL, -- creation timestamp - key_id VARCHAR NOT NULL, -- name/key-id of this key pair. for use in JWTs etc. - group_name VARCHAR, - is_default_pair BOOLEAN DEFAULT FALSE, -- whether this keypair is the default one for a participant context - use_duration BIGINT, -- maximum time this keypair can be active before it gets rotated - rotation_duration BIGINT, -- duration during which this keypair is in a transitional state (rotated, not yet deactivated) - serialized_public_key VARCHAR NOT NULL, -- serialized public key (PEM, JWK,...) - private_key_alias VARCHAR NOT NULL, -- alias under which the private key is stored in the HSM/Vault - state INT NOT NULL DEFAULT 100, -- KeyPairState - key_context VARCHAR --the key context, will end up in the VerificationMethod of the DID Document -); - -CREATE TABLE IF NOT EXISTS participant_context -( - participant_id VARCHAR PRIMARY KEY NOT NULL, -- ID of the ParticipantContext - created_date BIGINT NOT NULL, -- POSIX timestamp of the creation of the PC - last_modified_date BIGINT, -- POSIX timestamp of the last modified date - state INTEGER NOT NULL, -- 0 = CREATED, 1 = ACTIVE, 2 = DEACTIVATED - api_token_alias VARCHAR NOT NULL, -- alias under which this PC's api token is stored in the vault - did VARCHAR, -- the DID with which this participant is identified - roles JSON -- JSON array containing all the roles a user has. may be empty -); -CREATE UNIQUE INDEX IF NOT EXISTS participant_context_participant_id_uindex ON participant_context USING btree (participant_id); - diff --git a/deployment/consumer.tf b/deployment/consumer.tf index f4b78a69..89b348db 100644 --- a/deployment/consumer.tf +++ b/deployment/consumer.tf @@ -70,15 +70,11 @@ resource "kubernetes_config_map" "postgres-initdb-config-consumer" { } data = { "consumer-initdb-config.sql" = <<-EOT - CREATE USER consumer WITH ENCRYPTED PASSWORD 'consumer'; + CREATE USER consumer WITH ENCRYPTED PASSWORD 'consumer' SUPERUSER; CREATE DATABASE consumer; - \c consumer + \c consumer consumer - ${file("./assets/postgres/edc_schema.sql")} - ${file("./assets/postgres/ih_schema.sql")} - - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO consumer; EOT } } \ No newline at end of file diff --git a/deployment/modules/catalog-server/catalog-server.tf b/deployment/modules/catalog-server/catalog-server.tf index 60b3de60..829dc70c 100644 --- a/deployment/modules/catalog-server/catalog-server.tf +++ b/deployment/modules/catalog-server/catalog-server.tf @@ -145,6 +145,7 @@ resource "kubernetes_config_map" "catalog-server-config" { EDC_DATASOURCE_DEFAULT_URL = var.database.url EDC_DATASOURCE_DEFAULT_USER = var.database.user EDC_DATASOURCE_DEFAULT_PASSWORD = var.database.password + EDC_SQL_SCHEMA_AUTOCREATE = true # remote STS configuration EDC_IAM_STS_OAUTH_TOKEN_URL = var.sts-token-url diff --git a/deployment/modules/connector/controlplane.tf b/deployment/modules/connector/controlplane.tf index e83c42f8..25423449 100644 --- a/deployment/modules/connector/controlplane.tf +++ b/deployment/modules/connector/controlplane.tf @@ -175,6 +175,7 @@ resource "kubernetes_config_map" "connector-config" { EDC_DATASOURCE_DEFAULT_URL = var.database.url EDC_DATASOURCE_DEFAULT_USER = var.database.user EDC_DATASOURCE_DEFAULT_PASSWORD = var.database.password + EDC_SQL_SCHEMA_AUTOCREATE = true # remote STS configuration EDC_IAM_STS_OAUTH_TOKEN_URL = var.sts-token-url diff --git a/deployment/modules/connector/dataplane.tf b/deployment/modules/connector/dataplane.tf index 8acf5e49..90eebd60 100644 --- a/deployment/modules/connector/dataplane.tf +++ b/deployment/modules/connector/dataplane.tf @@ -127,6 +127,7 @@ resource "kubernetes_config_map" "dataplane-config" { EDC_DATASOURCE_DEFAULT_URL = var.database.url EDC_DATASOURCE_DEFAULT_USER = var.database.user EDC_DATASOURCE_DEFAULT_PASSWORD = var.database.password + EDC_SQL_SCHEMA_AUTOCREATE = true # remote STS configuration EDC_IAM_STS_OAUTH_TOKEN_URL = var.sts-token-url diff --git a/deployment/modules/identity-hub/main.tf b/deployment/modules/identity-hub/main.tf index a7036245..4bf9c4d3 100644 --- a/deployment/modules/identity-hub/main.tf +++ b/deployment/modules/identity-hub/main.tf @@ -160,6 +160,7 @@ resource "kubernetes_config_map" "identityhub-config" { EDC_DATASOURCE_DEFAULT_URL = var.database.url EDC_DATASOURCE_DEFAULT_USER = var.database.user EDC_DATASOURCE_DEFAULT_PASSWORD = var.database.password + EDC_SQL_SCHEMA_AUTOCREATE = true } } diff --git a/deployment/provider.tf b/deployment/provider.tf index 72e779ba..66f5f744 100644 --- a/deployment/provider.tf +++ b/deployment/provider.tf @@ -105,13 +105,10 @@ resource "kubernetes_config_map" "postgres-initdb-config-cs" { } data = { "cs-initdb-config.sql" = <<-EOT - CREATE USER catalog_server WITH ENCRYPTED PASSWORD 'catalog_server'; + CREATE USER catalog_server WITH ENCRYPTED PASSWORD 'catalog_server' SUPERUSER; CREATE DATABASE catalog_server; \c catalog_server - ${file("./assets/postgres/edc_schema.sql")} - - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO catalog_server; EOT } } @@ -123,13 +120,10 @@ resource "kubernetes_config_map" "postgres-initdb-config-pqna" { } data = { "provider-qna-initdb-config.sql" = <<-EOT - CREATE USER qna WITH ENCRYPTED PASSWORD 'provider-qna'; + CREATE USER qna WITH ENCRYPTED PASSWORD 'provider-qna' SUPERUSER; CREATE DATABASE provider_qna; \c provider_qna - ${file("./assets/postgres/edc_schema.sql")} - - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO qna; EOT } } @@ -141,13 +135,10 @@ resource "kubernetes_config_map" "postgres-initdb-config-pm" { } data = { "provider-manufacturing-initdb-config.sql" = <<-EOT - CREATE USER manufacturing WITH ENCRYPTED PASSWORD 'provider-manufacturing'; + CREATE USER manufacturing WITH ENCRYPTED PASSWORD 'provider-manufacturing' SUPERUSER; CREATE DATABASE provider_manufacturing; \c provider_manufacturing - ${file("./assets/postgres/edc_schema.sql")} - - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO manufacturing; EOT } } @@ -159,13 +150,10 @@ resource "kubernetes_config_map" "postgres-initdb-config-ih" { } data = { "ih-initdb-config.sql" = <<-EOT - CREATE USER identityhub WITH ENCRYPTED PASSWORD 'identityhub'; + CREATE USER identityhub WITH ENCRYPTED PASSWORD 'identityhub' SUPERUSER; CREATE DATABASE identityhub; \c identityhub - ${file("./assets/postgres/ih_schema.sql")} - - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO identityhub; EOT } } \ No newline at end of file