From 6cc5acd4ca69c394dfff2b4923a1e8acc737af81 Mon Sep 17 00:00:00 2001 From: Rajeev Date: Tue, 21 May 2024 15:54:27 +0530 Subject: [PATCH] V2 dpsetup (#265) * MCPODS-7636: * Use latest code deploy setup for v2 * Use naksha-hub service instead of xyz-hub * Use AWS secret key path with v2 suffix * in start-app.sh use service.namespace=Naksha-v2-${EC2_ENV_UPPER} * renamed xyz-hub.service to naksha-hub.service * MCPODS-7636: parameterise extension config * MCPODS-7636: pipeline changes * MCPODS-7636: pipeline changes * MCPODS-7636: SED command change * MCPODS-7636: pipeline changes * MCPODS-7636: swagger url change * MCPODS-7636: swagger url changes * MCPODS-7636: pipeline changes * MCPODS-7636: Add Env and Ext jar name in log * MCPODS-7636: remove v2_dpsetup branch from pipeline * MCPODS-7636: review comments fixed --------- Co-authored-by: Rajeev Jain --- .github/workflows/main-dev-deploy.yml | 3 ++- .github/workflows/main-e2e-deploy.yml | 3 ++- .github/workflows/main-prd-deploy.yml | 6 +++--- .github/workflows/reusable-s3-codedeploy-release.yml | 8 ++++++++ deployment/codedeploy/appspec.yml | 6 +++--- .../contents/naksha-hub/.config/cloud-config.json | 2 +- .../{xyz-hub.service => naksha-hub.service} | 2 +- .../codedeploy/contents/naksha-hub/set-app-params.sh | 2 +- .../codedeploy/contents/naksha-hub/start-app.sh | 2 +- deployment/codedeploy/scripts/ApplicationStart.sh | 2 +- deployment/codedeploy/scripts/ApplicationStop.sh | 8 ++++---- .../src/main/resources/swagger/openapi.yaml | 2 +- .../here/naksha/lib/core/models/ExtensionConfig.java | 12 +++++++++++- .../here/naksha/lib/extmanager/ExtensionCache.java | 6 ++++-- .../com/here/naksha/lib/extmanager/BaseSetup.java | 2 +- .../main/java/com/here/naksha/lib/hub/NakshaHub.java | 3 ++- 16 files changed, 46 insertions(+), 23 deletions(-) rename deployment/codedeploy/contents/naksha-hub/{xyz-hub.service => naksha-hub.service} (95%) diff --git a/.github/workflows/main-dev-deploy.yml b/.github/workflows/main-dev-deploy.yml index f62190178..086f6eaee 100644 --- a/.github/workflows/main-dev-deploy.yml +++ b/.github/workflows/main-dev-deploy.yml @@ -37,6 +37,7 @@ jobs: s3-bucket-name: ${{ vars.RELEASE_S3_BUCKET_NAME }} s3-bucket-dir-path: ${{ needs.Build-and-Publish.outputs.app-version }} s3-artifact-version: ${{ needs.Build-and-Publish.outputs.codedeploy-artifact-version }} + extension-s3-bucket-name: naksha-pvt-releases secrets: aws-key: ${{ secrets.AWS_KEY }} aws-secret: ${{ secrets.AWS_SECRET }} @@ -49,7 +50,7 @@ jobs: uses: ./.github/workflows/reusable-codedeploy-deployment.yml with: aws-region: ${{ vars.AWS_REGION }} - codedeploy-app-name: ${{ vars.CODEDEPLOY_APP_NAME }} + codedeploy-app-name: Naksha_v2_App codedeploy-group-name: ${{ vars.CODEDEPLOY_DEV_GROUP_NAME }} deployment-description: 'Deployment triggered by ${{ github.triggering_actor }} from Github repo [${{ github.repository }}], ${{ github.ref_type }} [${{ github.ref_name }}], commit sha [${{ github.sha }}]' s3-bucket-name: ${{ vars.RELEASE_S3_BUCKET_NAME }} diff --git a/.github/workflows/main-e2e-deploy.yml b/.github/workflows/main-e2e-deploy.yml index 5d0b930fc..73bd06959 100644 --- a/.github/workflows/main-e2e-deploy.yml +++ b/.github/workflows/main-e2e-deploy.yml @@ -33,6 +33,7 @@ jobs: s3-bucket-name: ${{ vars.RELEASE_S3_BUCKET_NAME }} s3-bucket-dir-path: ${{ needs.Build-and-Publish.outputs.app-version }} s3-artifact-version: ${{ needs.Build-and-Publish.outputs.codedeploy-artifact-version }} + extension-s3-bucket-name: naksha-pvt-releases secrets: aws-key: ${{ secrets.AWS_KEY }} aws-secret: ${{ secrets.AWS_SECRET }} @@ -45,7 +46,7 @@ jobs: uses: ./.github/workflows/reusable-codedeploy-deployment.yml with: aws-region: ${{ vars.AWS_REGION }} - codedeploy-app-name: ${{ vars.CODEDEPLOY_APP_NAME }} + codedeploy-app-name: Naksha_v2_App codedeploy-group-name: ${{ vars.CODEDEPLOY_E2E_GROUP_NAME }} deployment-description: 'Deployment triggered by ${{ github.triggering_actor }} from Github repo [${{ github.repository }}], ${{ github.ref_type }} [${{ github.ref_name }}], commit sha [${{ github.sha }}]' s3-bucket-name: ${{ vars.RELEASE_S3_BUCKET_NAME }} diff --git a/.github/workflows/main-prd-deploy.yml b/.github/workflows/main-prd-deploy.yml index 60139a9e0..be79469a3 100644 --- a/.github/workflows/main-prd-deploy.yml +++ b/.github/workflows/main-prd-deploy.yml @@ -4,8 +4,7 @@ name: Prod Workflow on: push: tags: - - '2.[0-9]+.[0-9]+' - - '!*rc*' + - '2.[0-9]+.[0-9]+-preprod' permissions: checks: write # for junit reporting @@ -34,6 +33,7 @@ jobs: s3-bucket-name: ${{ vars.RELEASE_S3_BUCKET_NAME }} s3-bucket-dir-path: ${{ needs.Build-and-Publish.outputs.app-version }} s3-artifact-version: ${{ needs.Build-and-Publish.outputs.codedeploy-artifact-version }} + extension-s3-bucket-name: ${{ vars.PRD_PVT_RELEASE_S3_BUCKET_NAME }} secrets: aws-key: ${{ secrets.AWS_KEY }} aws-secret: ${{ secrets.AWS_SECRET }} @@ -46,7 +46,7 @@ jobs: uses: ./.github/workflows/reusable-codedeploy-deployment.yml with: aws-region: ${{ vars.PRD_AWS_REGION }} - codedeploy-app-name: ${{ vars.CODEDEPLOY_APP_NAME }} + codedeploy-app-name: Naksha_v2_App codedeploy-group-name: ${{ vars.CODEDEPLOY_PRD_GROUP_NAME }} deployment-description: 'Deployment triggered by ${{ github.triggering_actor }} from Github repo [${{ github.repository }}], ${{ github.ref_type }} [${{ github.ref_name }}], commit sha [${{ github.sha }}]' # we use Prod access point (eu-west-1) to fetch deployment artifacts from E2E S3 bucket (us-east-1) diff --git a/.github/workflows/reusable-s3-codedeploy-release.yml b/.github/workflows/reusable-s3-codedeploy-release.yml index eb6bde68e..e04596d33 100644 --- a/.github/workflows/reusable-s3-codedeploy-release.yml +++ b/.github/workflows/reusable-s3-codedeploy-release.yml @@ -21,6 +21,10 @@ on: type: string description: 'A version (or suffix) to be used in artifact name while uploading it to AWS S3 bucket' required: true + extension-s3-bucket-name: + type: string + description: 'AWS S3 bucket name, where extension artifacts are uploaded' + required: true secrets: aws-key: description: 'AWS Access key of a user, who is allowed to upload artifact to S3 bucket' @@ -55,6 +59,10 @@ jobs: run: | BUNDLE_NAME=deployment_${{ inputs.s3-artifact-version }}.tar.gz echo "name=$BUNDLE_NAME" >> "$GITHUB_OUTPUT" + - name: Set extension s3 bucket name in cloud-config.json + run: | + export ACTUAL_BUCKET=${{ inputs.extension-s3-bucket-name }} + sed -i "s+SOME_S3_BUCKET+${ACTUAL_BUCKET}+g" download/contents/naksha-hub/.config/cloud-config.json - name: Preparing S3 artifact bundle run: | cd download diff --git a/deployment/codedeploy/appspec.yml b/deployment/codedeploy/appspec.yml index c15d66b37..08548efb4 100644 --- a/deployment/codedeploy/appspec.yml +++ b/deployment/codedeploy/appspec.yml @@ -26,7 +26,7 @@ file_exists_behavior: OVERWRITE files: - source: /contents destination: /home/admin - - source: /contents/naksha-hub/xyz-hub.service + - source: /contents/naksha-hub/naksha-hub.service destination: /etc/systemd/system # For deployments to Amazon Linux, Ubuntu Server, or RHEL instances, # you can specify a "permissions" @@ -68,9 +68,9 @@ permissions: mode: 755 type: - file - # change permission for file xyz-hub.service + # change permission for file naksha-hub.service - object: /etc/systemd/system - pattern: "xyz-hub.service" + pattern: "naksha-hub.service" owner: admin group: admin type: diff --git a/deployment/codedeploy/contents/naksha-hub/.config/cloud-config.json b/deployment/codedeploy/contents/naksha-hub/.config/cloud-config.json index 81a83670e..eeb966d2d 100644 --- a/deployment/codedeploy/contents/naksha-hub/.config/cloud-config.json +++ b/deployment/codedeploy/contents/naksha-hub/.config/cloud-config.json @@ -9,6 +9,6 @@ "extensionConfigParams": { "whitelistClasses": [ "java.*", "javax.*", "com.here.*", "jdk.internal.reflect.*", "com.sun.*", "org.w3c.dom.*", "sun.misc.*"], "intervalms": 30000, - "extensionsRootPath": "s3://naksha-pvt-releases/extensions/" + "extensionsRootPath": "s3://SOME_S3_BUCKET/extensions/" } } \ No newline at end of file diff --git a/deployment/codedeploy/contents/naksha-hub/xyz-hub.service b/deployment/codedeploy/contents/naksha-hub/naksha-hub.service similarity index 95% rename from deployment/codedeploy/contents/naksha-hub/xyz-hub.service rename to deployment/codedeploy/contents/naksha-hub/naksha-hub.service index 93088c7f8..51e65dd71 100644 --- a/deployment/codedeploy/contents/naksha-hub/xyz-hub.service +++ b/deployment/codedeploy/contents/naksha-hub/naksha-hub.service @@ -10,7 +10,7 @@ Description=Naksha-Hub [Service] User=admin Type=simple -PIDFile=/run/xyz-hub.pid +PIDFile=/run/naksha-hub.pid WorkingDirectory=/home/admin/naksha-hub ExecStart=/home/admin/naksha-hub/start-app.sh # ExecReload=/home/transang/startup.sh reload diff --git a/deployment/codedeploy/contents/naksha-hub/set-app-params.sh b/deployment/codedeploy/contents/naksha-hub/set-app-params.sh index 1ba626686..d2d14dd02 100644 --- a/deployment/codedeploy/contents/naksha-hub/set-app-params.sh +++ b/deployment/codedeploy/contents/naksha-hub/set-app-params.sh @@ -2,7 +2,7 @@ ### Set env params based on App secrets fetched from AWS Secrets Manager -SECRET_ID="$EC2_ENV/heremap/service/naksha" +SECRET_ID="$EC2_ENV/heremap/service/naksha-v2" SECRET_RESPONSE_JSON=`aws secretsmanager get-secret-value --region $EC2_REGION --secret-id $SECRET_ID` # Validate that we really got the successful response diff --git a/deployment/codedeploy/contents/naksha-hub/start-app.sh b/deployment/codedeploy/contents/naksha-hub/start-app.sh index 4ee3c1e8c..734e282dd 100755 --- a/deployment/codedeploy/contents/naksha-hub/start-app.sh +++ b/deployment/codedeploy/contents/naksha-hub/start-app.sh @@ -19,7 +19,7 @@ if [ ! -d logs ]; then fi # Set local parameters -export OTEL_RESOURCE_ATTRIBUTES=service.name=${EC2_INSTANCE_NAME},service.namespace=Naksha-${EC2_ENV_UPPER} +export OTEL_RESOURCE_ATTRIBUTES=service.name=${EC2_INSTANCE_NAME},service.namespace=Naksha-v2-${EC2_ENV_UPPER} # Print basic parameters (avoid printing secrets) echo "NAKSHA_CONFIG_PATH : $NAKSHA_CONFIG_PATH" diff --git a/deployment/codedeploy/scripts/ApplicationStart.sh b/deployment/codedeploy/scripts/ApplicationStart.sh index 35fda7a9d..3afeb99d8 100644 --- a/deployment/codedeploy/scripts/ApplicationStart.sh +++ b/deployment/codedeploy/scripts/ApplicationStart.sh @@ -2,5 +2,5 @@ # Start application echo "Starting Naksha service..." -sudo systemctl start xyz-hub +sudo systemctl start naksha-hub echo "Start triggered!" diff --git a/deployment/codedeploy/scripts/ApplicationStop.sh b/deployment/codedeploy/scripts/ApplicationStop.sh index 4f8e5e2f9..fc5947d45 100644 --- a/deployment/codedeploy/scripts/ApplicationStop.sh +++ b/deployment/codedeploy/scripts/ApplicationStop.sh @@ -2,10 +2,10 @@ # Stop Naksha service (only if it was configured) CODE=0 -sudo systemctl list-unit-files --all | grep -Fq "xyz-hub.service" || CODE=1 +sudo systemctl list-unit-files --all | grep -Fq "naksha-hub.service" || CODE=1 if [[ $CODE -eq 0 ]]; then - echo ">> Stopping service [xyz-hub]..." - sudo systemctl stop xyz-hub + echo ">> Stopping service [naksha-hub]..." + sudo systemctl stop naksha-hub else - echo ">> Service [xyz-hub] not found. Nothing to stop" + echo ">> Service [naksha-hub] not found. Nothing to stop" fi diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index 880e3305d..ef3754057 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -7,7 +7,7 @@ servers: # description: "PRD" #- url: "https://naksha-e2e.ext.mapcreator.here.com/" # description: "E2E" - - url: "https://naksha-dev.ext.mapcreator.here.com/" + - url: "https://naksha-v2-dev.ext.mapcreator.here.com/" description: "DEV" info: diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java index f91d0e759..b667723cd 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java @@ -32,6 +32,7 @@ public class ExtensionConfig { public static final String EXPIRY = "expiry"; public static final String EXTENSIONS = "extensions"; public static final String WHITELIST_DELEGATE_CLASSES = "whitelistDelegateClasses"; + public static final String ENV_NAME = "env"; @JsonProperty(EXPIRY) long expiry; @@ -42,14 +43,19 @@ public class ExtensionConfig { @JsonProperty(WHITELIST_DELEGATE_CLASSES) List whitelistDelegateClasses; + @JsonProperty(ENV_NAME) + String env; + @JsonCreator public ExtensionConfig( @JsonProperty(EXPIRY) @NotNull Long expiry, @JsonProperty(EXTENSIONS) @Nullable List extensions, - @JsonProperty(WHITELIST_DELEGATE_CLASSES) @Nullable List whitelistDelegateClasses) { + @JsonProperty(WHITELIST_DELEGATE_CLASSES) @Nullable List whitelistDelegateClasses, + @JsonProperty(ENV_NAME) @NotNull String env) { this.expiry = expiry; this.extensions = extensions; this.whitelistDelegateClasses = whitelistDelegateClasses; + this.env = env; } public long getExpiry() { @@ -63,4 +69,8 @@ public List getExtensions() { public List getWhilelistDelegateClass() { return whitelistDelegateClasses; } + + public String getEnv() { + return env; + } } diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index dea2ec6ff..51334abc4 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -134,9 +134,11 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex loaderCache.put(extension.getId(), new KVPair(extension, loader)); PluginCache.removeExtensionCache(extension.getId()); logger.info( - "Extension {},{} is successfully loaded into the cache.", + "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", extension.getId(), - extension.getVersion()); + extension.getVersion(), + extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1), + extensionConfig.getEnv()); } } diff --git a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java index fc5b546e1..f997e0f67 100644 --- a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java +++ b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java @@ -25,6 +25,6 @@ public ExtensionConfig getExtensionConfig() { } catch (IOException e) { throw new RuntimeException(e); } - return new ExtensionConfig(System.currentTimeMillis() + 6000, list,whitelistUrls); + return new ExtensionConfig(System.currentTimeMillis() + 6000, list,whitelistUrls,"test"); } } diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index 305b5cf62..8847f2c27 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -300,7 +300,8 @@ private static WriteXyzCollections createAdminCollectionsRequest() { return new ExtensionConfig( System.currentTimeMillis() + extensionConfigParams.getIntervalMs(), extList, - extensionConfigParams.getWhiteListClasses()); + extensionConfigParams.getWhiteListClasses(), + this.nakshaHubConfig.env.toLowerCase()); } private List loadExtensionConfigFromS3(String extensionRootPath) {