Skip to content

Commit

Permalink
feat: add standalong dataplanes for K8S (#281)
Browse files Browse the repository at this point in the history
* add standalone dataplane runtime

* add k8s deployment

* fix configuration

* fix health check, clean up variables

* fix superuser seed, kubernetes probes

* formatting

* removed codeql

* fix test
  • Loading branch information
paullatzelsperger authored Jul 11, 2024
1 parent a5f00bf commit 7f23c0c
Show file tree
Hide file tree
Showing 26 changed files with 388 additions and 130 deletions.
68 changes: 0 additions & 68 deletions .github/workflows/codeql-analysis.yml

This file was deleted.

2 changes: 1 addition & 1 deletion .github/workflows/run-terraform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
cluster_name: dcp-demo

- name: "Load runtime images into KinD"
run: kind load docker-image connector:latest identity-hub:latest catalog-server:latest -n dcp-demo
run: kind load docker-image controlplane:latest identity-hub:latest catalog-server:latest -n dcp-demo

- name: "Install nginx ingress controller"
run: |-
Expand Down
2 changes: 1 addition & 1 deletion .run/Connector Consumer Corp.run.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<option name="ALTERNATIVE_JRE_PATH" value="temurin-21" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
<option name="MAIN_CLASS_NAME" value="org.eclipse.edc.boot.system.runtime.BaseRuntime" />
<module name="mvd.launchers.connector.main" />
<module name="mvd.launchers.controlplane.main" />
<option name="VM_PARAMETERS" value="-Dedc.iam.issuer.id=&quot;did:web:localhost%3A7083&quot; -Dweb.http.port=8080 -Dweb.http.path=&quot;/api&quot; -Dweb.http.management.port=8081 -Dweb.http.management.path=&quot;/api/management/&quot; -Dweb.http.protocol.port=8082 -Dweb.http.protocol.path=&quot;/api/dsp&quot; -Dweb.http.control.port=8083 -Dweb.http.control.path=&quot;/api/control&quot; -Dweb.http.catalog.port=8084 -Dweb.http.catalog.path=&quot;/api/catalog&quot; -Dweb.http.version.port=8085 -Dweb.http.version.path=&quot;/api/version&quot; -Dedc.api.auth.key=&quot;password&quot; -Dedc.iam.did.web.use.https=&quot;false&quot; -Dedc.iam.sts.privatekey.alias=&quot;did:web:localhost%3A7083-alias&quot; -Dedc.iam.sts.publickey.id=&quot;did:web:localhost%3A7083#key-1&quot; -Dedc.dsp.callback.address=&quot;http://localhost:8082/api/dsp&quot; -Dedc.participant.id=&quot;did:web:localhost%3A7083&quot; -Dedc.catalog.cache.execution.delay.seconds=5 -Dedc.catalog.cache.execution.period.seconds=10 -Dedc.mvd.participants.list.file=&quot;deployment/assets/participants/participants.local.json&quot;" />
<method v="2">
<option name="Make" enabled="true" />
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ All commands are executed from the **repository's root folder** unless stated ot
./gradlew dockerize -Ppersistence=true
```

this builds the runtime images and creates the following docker images: `connector:latest`, `catalog-server:latest`
this builds the runtime images and creates the following docker images: `controlplane:latest`, `catalog-server:latest`
and `identity-hub:latest` in the local docker image cache. Note the `-Ppersistence` flag which puts the HashiCorp Vault
module and PostgreSQL persistence modules on the classpath. These obviously require additional configuration, which is
handled by the Terraform scripts.
Expand All @@ -136,7 +136,7 @@ Next, we bring up and configure the Kubernetes Cluster
kind create cluster -n dcp-demo --config deployment/kind.config.yaml

# Load docker images into KinD
kind load docker-image connector:latest identity-hub:latest catalog-server:latest -n dcp-demo
kind load docker-image controlplane:latest identity-hub:latest catalog-server:latest -n dcp-demo

# Deploy an NGINX ingress
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
Expand Down
5 changes: 3 additions & 2 deletions deployment/consumer.tf
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,13 @@ module "consumer-connector" {
source = "./modules/connector"
humanReadableName = "consumer"
participantId = var.consumer-did
participant-did = var.consumer-did
database = {
user = "consumer"
password = "consumer"
url = "jdbc:postgresql://${module.consumer-postgres.database-url}/consumer"
}
namespace = kubernetes_namespace.ns.metadata.0.name
vault-url = "http://consumer-vault:8200"
namespace = kubernetes_namespace.ns.metadata.0.name
}

# consumer identity hub
Expand All @@ -43,12 +42,14 @@ module "consumer-identityhub" {
password = "consumer"
url = "jdbc:postgresql://${module.consumer-postgres.database-url}/consumer"
}
namespace = kubernetes_namespace.ns.metadata.0.name
}

# consumer vault
module "consumer-vault" {
source = "./modules/vault"
humanReadableName = "consumer-vault"
namespace = kubernetes_namespace.ns.metadata.0.name
}

# Postgres database for the consumer
Expand Down
20 changes: 19 additions & 1 deletion deployment/modules/catalog-server/catalog-server.tf
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,25 @@ resource "kubernetes_deployment" "connector" {

liveness_probe {
exec {
command = ["curl", "-X POST", "http://localhost:8080/api/check/health"]
command = ["curl", "-X GET", "http://localhost:${var.ports.web}/api/check/liveness"]
}
failure_threshold = 10
period_seconds = 5
timeout_seconds = 30
}

readiness_probe {
exec {
command = ["curl", "-X GET", "http://localhost:${var.ports.web}/api/check/readiness"]
}
failure_threshold = 10
period_seconds = 5
timeout_seconds = 30
}

startup_probe {
exec {
command = ["curl", "-X GET", "http://localhost:${var.ports.web}/api/check/startup"]
}
failure_threshold = 10
period_seconds = 5
Expand Down
3 changes: 1 addition & 2 deletions deployment/modules/catalog-server/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,7 @@ variable "participant-did" {
}

variable "namespace" {
type = string
default = "mvd"
type = string
}

variable "ports" {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,34 +17,34 @@
# SPDX-License-Identifier: Apache-2.0
#

resource "kubernetes_deployment" "connector" {
resource "kubernetes_deployment" "controlplane" {
metadata {
name = "${lower(var.humanReadableName)}-connector"
name = "${lower(var.humanReadableName)}-controlplane"
namespace = var.namespace
labels = {
App = "${lower(var.humanReadableName)}-connector"
App = "${lower(var.humanReadableName)}-controlplane"
}
}

spec {
replicas = 1
selector {
match_labels = {
App = "${lower(var.humanReadableName)}-connector"
App = "${lower(var.humanReadableName)}-controlplane"
}
}

template {
metadata {
labels = {
App = "${lower(var.humanReadableName)}-connector"
App = "${lower(var.humanReadableName)}-controlplane"
}
}

spec {
container {
name = "connector-${lower(var.humanReadableName)}"
image = "connector:latest"
image = "controlplane:latest"
image_pull_policy = "Never"

env_from {
Expand All @@ -68,7 +68,25 @@ resource "kubernetes_deployment" "connector" {

liveness_probe {
exec {
command = ["curl", "-X POST", "http://localhost:8080/api/check/health"]
command = ["curl", "-X GET", "http://localhost:${var.ports.web}/api/check/liveness"]
}
failure_threshold = 10
period_seconds = 5
timeout_seconds = 30
}

readiness_probe {
exec {
command = ["curl", "-X GET", "http://localhost:${var.ports.web}/api/check/readiness"]
}
failure_threshold = 10
period_seconds = 5
timeout_seconds = 30
}

startup_probe {
exec {
command = ["curl", "-X GET", "http://localhost:${var.ports.web}/api/check/startup"]
}
failure_threshold = 10
period_seconds = 5
Expand Down Expand Up @@ -118,17 +136,18 @@ resource "kubernetes_config_map" "participants-map" {

resource "kubernetes_config_map" "connector-config" {
metadata {
name = "${lower(var.humanReadableName)}-connector-config"
name = "${lower(var.humanReadableName)}-controlplane-config"
namespace = var.namespace
}

## Create databases for keycloak and MIW, create users and assign privileges
data = {
EDC_PARTICIPANT_ID = var.participantId
EDC_API_AUTH_KEY = "password"
EDC_IAM_ISSUER_ID = var.participant-did
EDC_IAM_ISSUER_ID = var.participantId
EDC_IAM_DID_WEB_USE_HTTPS = false
WEB_HTTP_PORT = var.ports.web
WEB_HTTP_PATH = "/"
WEB_HTTP_PATH = "/api"
WEB_HTTP_MANAGEMENT_PORT = var.ports.management
WEB_HTTP_MANAGEMENT_PATH = "/api/management"
WEB_HTTP_CONTROL_PORT = var.ports.control
Expand All @@ -140,10 +159,9 @@ resource "kubernetes_config_map" "connector-config" {
EDC_API_AUTH_KEY = "password"
EDC_DSP_CALLBACK_ADDRESS = "http://${local.controlplane-service-name}:${var.ports.protocol}/api/dsp"
EDC_IAM_STS_PRIVATEKEY_ALIAS = var.aliases.sts-private-key
EDC_IAM_STS_PUBLICKEY_ID = "${var.participant-did}#${var.aliases.sts-public-key-id}"
EDC_IAM_STS_PUBLICKEY_ID = "${var.participantId}#${var.aliases.sts-public-key-id}"
JAVA_TOOL_OPTIONS = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=${var.ports.debug}"
EDC_IH_AUDIENCE_REGISTRY_PATH = "/etc/registry/registry.json"
EDC_PARTICIPANT_ID = var.participantId
EDC_VAULT_HASHICORP_URL = var.vault-url
EDC_VAULT_HASHICORP_TOKEN = var.vault-token
EDC_MVD_PARTICIPANTS_LIST_FILE = "/etc/participants/participants.json"
Expand Down
123 changes: 123 additions & 0 deletions deployment/modules/connector/dataplane.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
#
# Copyright (c) 2023 Contributors to the Eclipse Foundation
#
# See the NOTICE file(s) distributed with this work for additional
# information regarding copyright ownership.
#
# This program and the accompanying materials are made available under the
# terms of the Apache License, Version 2.0 which is available at
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# SPDX-License-Identifier: Apache-2.0
#

resource "kubernetes_deployment" "dataplane" {
# needs a hard dependency, otherwise the dataplane registration fails, and it is not retried
depends_on = [kubernetes_deployment.controlplane]
metadata {
name = "${lower(var.humanReadableName)}-dataplane"
namespace = var.namespace
labels = {
App = "${lower(var.humanReadableName)}-dataplane"
}
}

spec {
replicas = 1
selector {
match_labels = {
App = "${lower(var.humanReadableName)}-dataplane"
}
}

template {
metadata {
labels = {
App = "${lower(var.humanReadableName)}-dataplane"
}
}

spec {
container {
name = "dataplane-${lower(var.humanReadableName)}"
image = "dataplane:latest"
image_pull_policy = "Never"

env_from {
config_map_ref {
name = kubernetes_config_map.dataplane-config.metadata[0].name
}
}

port {
container_port = var.ports.public
name = "public-port"
}

port {
container_port = var.ports.debug
name = "debug-port"
}

liveness_probe {
exec {
command = ["curl", "-X GET", "http://localhost:${var.ports.web}/api/check/liveness"]
}
failure_threshold = 10
period_seconds = 5
timeout_seconds = 30
}

readiness_probe {
exec {
command = ["curl", "-X GET", "http://localhost:${var.ports.web}/api/check/readiness"]
}
failure_threshold = 10
period_seconds = 5
timeout_seconds = 30
}

startup_probe {
exec {
command = ["curl", "-X GET", "http://localhost:${var.ports.web}/api/check/startup"]
}
failure_threshold = 10
period_seconds = 5
timeout_seconds = 30
}
}
}
}
}
}

resource "kubernetes_config_map" "dataplane-config" {
metadata {
name = "${lower(var.humanReadableName)}-dataplane-config"
namespace = var.namespace
}

## Create databases for keycloak and MIW, create users and assign privileges
data = {
# hostname is "localhost" by default, but must be the service name at which the dataplane is reachable. URL scheme and port are appended by the application
EDC_HOSTNAME = local.dataplane-service-name
EDC_RUNTIME_ID = "${var.humanReadableName}-dataplane"
EDC_PARTICIPANT_ID = var.participantId
EDC_TRANSFER_PROXY_TOKEN_VERIFIER_PUBLICKEY_ALIAS = "${var.participantId}#${var.aliases.sts-public-key-id}"
EDC_TRANSFER_PROXY_TOKEN_SIGNER_PRIVATEKEY_ALIAS = var.aliases.sts-private-key
EDC_DPF_SELECTOR_URL = "http://${local.controlplane-service-name}:${var.ports.control}/api/control/v1/dataplanes"
WEB_HTTP_PORT = var.ports.web
WEB_HTTP_PATH = "/api"
WEB_HTTP_CONTROL_PORT = var.ports.control
WEB_HTTP_CONTROL_PATH = "/api/control"
WEB_HTTP_PUBLIC_PORT = var.ports.public
WEB_HTTP_PUBLIC_PATH = "/api/public"
JAVA_TOOL_OPTIONS = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=${var.ports.debug}"
}
}
Loading

0 comments on commit 7f23c0c

Please sign in to comment.