diff --git a/.github/workflows/run-terraform.yml b/.github/workflows/run-e2e-tests.yml similarity index 94% rename from .github/workflows/run-terraform.yml rename to .github/workflows/run-e2e-tests.yml index 8130d2ba..ea5690ef 100644 --- a/.github/workflows/run-terraform.yml +++ b/.github/workflows/run-e2e-tests.yml @@ -18,7 +18,7 @@ # --- -name: "Run DCP Demo locally" +name: "Execute E2E Tests" on: push: pull_request: @@ -36,7 +36,7 @@ concurrency: jobs: - Deploy-with-Terraform: + Run-E2E-Tests: runs-on: ubuntu-latest steps: @@ -101,6 +101,11 @@ jobs: run: |- ./gradlew -DincludeTags="EndToEndTest" test -DverboseTest=true + - name: "Print log if test failed" + if: failure() + run: |- + kubectl logs deployment/provider-qna-controlplane -n mvd + - name: "Destroy the KinD cluster" run: >- kind delete cluster -n dcp-demo \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4ab1d2f7..67d4a095 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -33,6 +33,7 @@ edc-ext-jsonld = { module = "org.eclipse.edc:json-ld", version.ref = "edc" } edc-api-dsp-config = { module = "org.eclipse.edc:dsp-http-api-configuration", version.ref = "edc" } edc-dcp = { module = "org.eclipse.edc:identity-trust-service", version.ref = "edc" } edc-controlplane-core = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } +edc-controlplane-transform = { module = "org.eclipse.edc:control-plane-transform", version.ref = "edc" } edc-controlplane-services = { module = "org.eclipse.edc:control-plane-aggregate-services", version.ref = "edc" } edc-config-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } edc-auth-tokenbased = { module = "org.eclipse.edc:auth-tokenbased", version.ref = "edc" } @@ -44,6 +45,7 @@ edc-api-management-asset = { module = "org.eclipse.edc:asset-api", version.ref = edc-api-management-edr = { module = "org.eclipse.edc:edr-cache-api", version.ref = "edc" } edc-api-management-policy = { module = "org.eclipse.edc:policy-definition-api", version.ref = "edc" } edc-api-management-contractdef = { module = "org.eclipse.edc:contract-definition-api", version.ref = "edc" } +edc-api-management-dataplaneselector = { module = "org.eclipse.edc:data-plane-selector-api", version.ref = "edc" } edc-api-observability = { module = "org.eclipse.edc:api-observability", version.ref = "edc" } edc-api-control-configuration = { module = "org.eclipse.edc:control-api-configuration", version.ref = "edc" } edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } @@ -69,6 +71,7 @@ edc-lib-jws2020 = { module = "org.eclipse.edc:jws2020-lib", version.ref = "edc" edc-lib-transform = { module = "org.eclipse.edc:transform-lib", version.ref = "edc" } edc-lib-crypto = { module = "org.eclipse.edc:crypto-common-lib", version.ref = "edc" } edc-lib-keys = { module = "org.eclipse.edc:keys-lib", version.ref = "edc" } +edc-lib-jsonld = { module = "org.eclipse.edc:json-ld-lib", version.ref = "edc" } # EDC dataplane client modules (used in controlplane) edc-dpf-transfer = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } @@ -161,7 +164,7 @@ dpf = ["edc-dpf-selector-core", "edc-spi-dataplane-selector", "edc-dpf-selector- connector = ["edc-boot", "edc-core-connector", "edc-ext-http", "edc-ext-observability", "edc-ext-jsonld"] -controlplane = ["edc-controlplane-core", "edc-config-filesystem", "edc-auth-tokenbased", "edc-auth-configuration", "edc-api-management", "edc-api-management-config","edc-api-management-edr", +controlplane = ["edc-controlplane-core", "edc-config-filesystem", "edc-auth-tokenbased", "edc-auth-configuration", "edc-api-management", "edc-api-management-config","edc-api-management-edr","edc-api-management-dataplaneselector", "edc-api-observability", "edc-dsp", "edc-spi-jwt", "edc-ext-http", "edc-controlplane-callback-dispatcher-event", "edc-controlplane-callback-dispatcher-http", "edc-identity-core-did", "edc-dcp-core", "edc-identity-trust-transform", "edc-api-control-configuration", "edc-lib-transform", "edc-identity-vc-ldp", "edc-did-web", "edc-lib-jws2020", "edc-core-edrstore", "edc-edr-storereceiver"] diff --git a/tests/end2end/build.gradle.kts b/tests/end2end/build.gradle.kts index d66843ed..3f9e8f99 100644 --- a/tests/end2end/build.gradle.kts +++ b/tests/end2end/build.gradle.kts @@ -23,4 +23,8 @@ dependencies { testImplementation(libs.parsson) testImplementation(libs.restAssured) testImplementation(libs.awaitility) + testImplementation(libs.edc.fc.core) + testImplementation(libs.edc.lib.transform) + testImplementation(libs.edc.lib.jsonld) + testImplementation(libs.edc.controlplane.transform) } diff --git a/tests/end2end/src/test/java/org/eclipse/edc/demo/tests/transfer/TransferEndToEndTest.java b/tests/end2end/src/test/java/org/eclipse/edc/demo/tests/transfer/TransferEndToEndTest.java index e853b000..674abe3c 100644 --- a/tests/end2end/src/test/java/org/eclipse/edc/demo/tests/transfer/TransferEndToEndTest.java +++ b/tests/end2end/src/test/java/org/eclipse/edc/demo/tests/transfer/TransferEndToEndTest.java @@ -14,11 +14,27 @@ package org.eclipse.edc.demo.tests.transfer; -import io.restassured.path.json.JsonPath; import io.restassured.specification.RequestSpecification; import jakarta.json.Json; +import jakarta.json.JsonArray; +import org.eclipse.edc.catalog.transform.JsonObjectToCatalogTransformer; +import org.eclipse.edc.catalog.transform.JsonObjectToDataServiceTransformer; +import org.eclipse.edc.catalog.transform.JsonObjectToDatasetTransformer; +import org.eclipse.edc.catalog.transform.JsonObjectToDistributionTransformer; +import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; +import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset; +import org.eclipse.edc.connector.controlplane.transform.odrl.OdrlTransformersFactory; +import org.eclipse.edc.jsonld.TitaniumJsonLd; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.jsonld.util.JacksonJsonLd; import org.eclipse.edc.junit.annotations.EndToEndTest; import org.eclipse.edc.junit.testfixtures.TestUtils; +import org.eclipse.edc.spi.agent.ParticipantIdMapper; +import org.eclipse.edc.spi.monitor.ConsoleMonitor; +import org.eclipse.edc.transform.TypeTransformerRegistryImpl; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.transform.transformer.edc.to.JsonValueToGenericTypeTransformer; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.time.Duration; @@ -43,11 +59,15 @@ public class TransferEndToEndTest { private static final String PROVIDER_DSP_URL = "http://provider-qna-controlplane:8082"; // DID of the provider company private static final String PROVIDER_ID = "did:web:provider-identityhub%3A7083:provider"; - // public API endpoint of the provider-qna connector, goes through the incress controller + // public API endpoint of the provider-qna connector, goes through the ingress controller private static final String PROVIDER_PUBLIC_URL = "http://127.0.0.1/provider-qna/public"; - private static final Duration TEST_TIMEOUT_DURATION = Duration.ofSeconds(30); + private static final String PROVIDER_MANAGEMENT_URL = "http://127.0.0.1/provider-qna/cp"; + private static final Duration TEST_TIMEOUT_DURATION = Duration.ofSeconds(120); private static final Duration TEST_POLL_DELAY = Duration.ofSeconds(2); + private final TypeTransformerRegistry transformerRegistry = new TypeTransformerRegistryImpl(); + private final JsonLd jsonLd = new TitaniumJsonLd(new ConsoleMonitor()); + private static RequestSpecification baseRequest() { return given() .header("X-Api-Key", "password") @@ -55,6 +75,26 @@ private static RequestSpecification baseRequest() { .when(); } + @BeforeEach + void setup() { + transformerRegistry.register(new JsonObjectToCatalogTransformer()); + transformerRegistry.register(new JsonObjectToDatasetTransformer()); + transformerRegistry.register(new JsonObjectToDataServiceTransformer()); + transformerRegistry.register(new JsonObjectToDistributionTransformer()); + transformerRegistry.register(new JsonValueToGenericTypeTransformer(JacksonJsonLd.createObjectMapper())); + OdrlTransformersFactory.jsonObjectToOdrlTransformers(new ParticipantIdMapper() { + @Override + public String toIri(String s) { + return s; + } + + @Override + public String fromIri(String s) { + return s; + } + }).forEach(transformerRegistry::register); + } + @Test void transferData() { var emptyQueryBody = Json.createObjectBuilder() @@ -66,18 +106,32 @@ void transferData() { await().atMost(TEST_TIMEOUT_DURATION) .pollDelay(TEST_POLL_DELAY) .untilAsserted(() -> { - var oid = baseRequest() + var jo = baseRequest() .body(emptyQueryBody) .post(CONSUMER_CATALOG_URL + "/api/catalog/v1alpha/catalog/query") .then() .log().ifError() .statusCode(200) - // yes, it's a bit brittle with the hardcoded indexes, but it appears to work. - .extract().body().asString(); - var jp = new JsonPath(oid).getString("[0]['dcat:dataset'][1]['dcat:dataset'][0]['odrl:hasPolicy']['@id']"); - - assertThat(jp).isNotNull(); - offerId.set(jp); + .extract().body().as(JsonArray.class); + + var offerIdsFiltered = jo.stream().map(jv -> { + + var expanded = jsonLd.expand(jv.asJsonObject()).orElseThrow(f -> new AssertionError(f.getFailureDetail())); + var cat = transformerRegistry.transform(expanded, Catalog.class).orElseThrow(f -> new AssertionError(f.getFailureDetail())); + return cat.getDatasets().stream().filter(ds -> ds instanceof Catalog) // filter for CatalogAssets + .map(ds -> (Catalog) ds) + .filter(sc -> sc.getDataServices().stream().anyMatch(dataService -> dataService.getEndpointUrl().contains("provider-qna"))) // filter for assets from the Q&A Provider + .flatMap(c -> c.getDatasets().stream()) + .filter(dataset -> dataset.getId().equals("asset-1")) // filter for the asset we're allowed to negotiate + .map(Dataset::getOffers) + .map(offers -> offers.keySet().iterator().next()) + .findFirst() + .orElse(null); + }).toList(); + assertThat(offerIdsFiltered).hasSize(1); + var oid = offerIdsFiltered.get(0); + assertThat(oid).isNotNull(); + offerId.set(oid); }); // initiate negotiation @@ -110,6 +164,21 @@ void transferData() { }); + // wait until provider's dataplane is available + await().atMost(TEST_TIMEOUT_DURATION) + .pollDelay(TEST_POLL_DELAY) + .untilAsserted(() -> { + var jp = baseRequest() + .get(PROVIDER_MANAGEMENT_URL + "/api/management/v3/dataplanes") + .then() + .statusCode(200) + .log().ifValidationFails() + .extract().body().jsonPath(); + + var state = jp.getString("state"); + assertThat(state).contains("AVAILABLE"); + }); + //start transfer process var tpRequest = TestUtils.getResourceFileContentAsString("transfer-request.json") .replace("{{PROVIDER_ID}}", PROVIDER_ID) @@ -124,7 +193,21 @@ void transferData() { .statusCode(200) .extract().body().jsonPath().getString("@id"); - // fetch EDR for transfer process + // wait until transfer process is in STARTED state + await().atMost(TEST_TIMEOUT_DURATION) + .pollDelay(TEST_POLL_DELAY) + .untilAsserted(() -> { + var jp = baseRequest() + .body(emptyQueryBody) + .post(CONSUMER_MANAGEMENT_URL + "/api/management/v3/transferprocesses/request") + .then() + .statusCode(200) + .extract().body().jsonPath(); + + assertThat(jp.getString("state")).contains("STARTED"); + }); + + // fetch EDR for transfer processs var endpoint = new AtomicReference(); var token = new AtomicReference(); await().atMost(TEST_TIMEOUT_DURATION) @@ -133,6 +216,7 @@ void transferData() { var jp = baseRequest() .get(CONSUMER_MANAGEMENT_URL + "/api/management/v3/edrs/%s/dataaddress".formatted(transferProcessId)) .then() + .log().ifValidationFails() .statusCode(200) .onFailMessage("Expected to find an EDR with transfer ID %s but did not!".formatted(transferProcessId)) .extract().body().jsonPath();