Address merge conflicts

ORNL · Oct 25, 2023 · ef6a21d · ef6a21d
2 parents e418c71 + b7f808e
commit ef6a21d
Show file tree

Hide file tree

Showing 38 changed files with 1,566 additions and 1,084 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -3,6 +3,10 @@
 
 # Tasks
 
-[ ] - Formatter has been run
-[ ] - CHANGELOG comment
-[ ] - Labels have been assigned to the pr
+* [ ] - A description of the PR has been provided, and a diagram included if it is a new feature.
+* [ ] - Formatter has been run
+* [ ] - CHANGELOG comment has been added
+* [ ] - Labels have been assigned to the pr
+* [ ] - A reviwer has been added
+* [ ] - A user has been assigned to work on the pr
+* [ ] - If new feature a unit test has been added
diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml
@@ -33,5 +33,5 @@ jobs:
         run: black --check .
       - name: Lint with flake8
         run: |
-          flake8 --count --select=E9,F63,F7,F82 --show-source --statistics .
-          flake8 --count --statistics .
+          flake8 --count --select=E9,F63,F7,F82 --show-source --statistics --max-line-length 100 .
+          flake8 --count --statistics --max-line-length 100 .
diff --git a/.gitignore b/.gitignore
@@ -15,6 +15,8 @@ repository/server/Version.hpp
 scripts/globus/datafed-home-repo.sh
 scripts/globus/datafed-home-repo-form.json
 scripts/globus/mapping.json
+scripts/admin_datafed_backup.sh
+scripts/admin_refresh_certs.sh
 services/
 web/SDMS.proto
 web/SDMS_Anon.proto

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
 
 stages:
   - setup
+  - clear-docker-cache
   - build-deploy-base
   - build-unit-test-deploy
   - end-to-end-setup
@@ -18,13 +19,28 @@ setup-vms:
     project: 8f4/datafedci
     branch: main
     strategy: depend
-#    - datafed-infrastructure
-#  script:
-#    - ./scripts/pipeline_setup.sh --compute-instance-name "ci-datafed-arangodb"
-#    - ./scripts/pipeline_setup.sh --compute-instance-name "ci-datafed-core"
-#    - ./scripts/pipeline_setup.sh --compute-instance-name "ci-datafed-globus1"
-#    - ./scripts/pipeline_setup.sh --compute-instance-name "ci-datafed-globus2"
-#    - ./scripts/pipeline_setup.sh --compute-instance-name "ci-datafed-client"
+
+################################################################################
+# STAGE: clear-docker-cache
+################################################################################
+# Used to clear out the cache on VMs where the images are being built
+clear-core-cache:
+  stage: clear-docker-cache
+  tags:
+    - ci_1
+  script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
+    - docker system prune -f
+    - ./scripts/ci_purge_images.sh
+
+clear-repo-cache:
+  stage: clear-docker-cache
+  tags:
+    - ci_3
+  script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
+    - docker system prune -f
+    - ./scripts/ci_purge_images.sh
 
 ################################################################################
 # STAGE: build-deploy-base
@@ -45,11 +61,13 @@ build-ws-base:
     - ci-datafed-core
     - docker
   script:
+    - docker system prune -f
     - docker build -f web/docker/Dockerfile.web-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} .
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
     - docker push code.ornl.gov:4567/dlsw/datafed/ws-base
 
 build-core-base:
+  needs: ["clear-core-cache"]
   stage: build-deploy-base
   variables:
     IMAGE_TAG: "dlsw/datafed/core-base"
@@ -58,11 +76,13 @@ build-core-base:
     - ci-datafed-core
     - docker
   script:
+    - docker system prune -f
     - docker build -f core/docker/Dockerfile.core-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} .
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
     - docker push code.ornl.gov:4567/dlsw/datafed/core-base
 
 build-repo-base:
+  needs: ["clear-repo-cache"]
   variables:
     IMAGE_TAG: "dlsw/datafed/repo-base"
     GIT_STRATEGY: clone
@@ -71,14 +91,17 @@ build-repo-base:
     - ci-datafed-repo
     - docker
   script:
+    - docker system prune -f
     - docker build -f repository/docker/Dockerfile.repo-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} .
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
     - docker push code.ornl.gov:4567/dlsw/datafed/repo-base
 
 
 build-gcs-base:
+  needs: ["clear-repo-cache"]
   variables:
     GIT_SUBMODULE_STRATEGY: recursive
+    GIT_STRATEGY: clone
     DISTRO: "ubuntu:focal"
     GCS_TAG: "code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
     IMAGE_TAG2: "dlsw/datafed/gcs-authz-base"
@@ -88,37 +111,18 @@ build-gcs-base:
     - ci-datafed-globus
     - docker
   script:
+    - docker system prune -f
     - cd external/globus-connect-server-deploy/docker
     - ./docker-build-gcs.sh "${DISTRO}"
     - LATEST_IMAGE=$(docker images | grep ubuntu-focal | head -n1 | awk '{print $3}')
     - docker tag "$LATEST_IMAGE" "$GCS_TAG" 
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
     - docker push "$GCS_TAG"
-    - cd ../../../
+    - cd ../../../ # back too root of project
     - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG2} .
     - docker push code.ornl.gov:4567/${IMAGE_TAG2}
     - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG3} .
     - docker push code.ornl.gov:4567/${IMAGE_TAG3}
-# This build runs two builds, one to build the gcs image provided by Globus
-# The second to build on top of the gcs image and add the authz dependencies
-#build-gcs-authz-base:
-#  variables:
-#    IMAGE_TAG: "dlsw/datafed/gcs-base"
-#    IMAGE_TAG2: "dlsw/datafed/gcs-authz-base"
-#    IMAGE_TAG3: "dlsw/datafed/gcs-authz"
-#    GIT_STRATEGY: clone
-#  stage: build-deploy-base
-#  tags:
-#    - ci-datafed-globus
-#    - docker
-#  script:
-#    - docker build -f repository/docker/Dockerfile.gcs-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} .
-#    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
-#    - docker push code.ornl.gov:4567/${IMAGE_TAG}
-#    - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG2} .
-#    - docker push code.ornl.gov:4567/${IMAGE_TAG2}
-#    - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG3} .
-#    - docker push code.ornl.gov:4567/${IMAGE_TAG3}
 
 ################################################################################
 # STAGE: build
@@ -271,15 +275,16 @@ end-to-end-ws-setup:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
     - mkdir -p "$HOST_LOG_FILE_PATH"
-    - chmod 777 "$HOST_LOG_FILE_PATH"
+    - chown gitlab-runner "$HOST_LOG_FILE_PATH"
     - ./scripts/generate_datafed.sh
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
     - CERT_FILE_NAME=$(basename "${CI_DATAFED_WEB_CERT_PATH}")
     - KEY_FILE_NAME=$(basename "${CI_DATAFED_WEB_KEY_PATH}")
     - ./scripts/container_stop.sh -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}"
+    - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
     - >
       docker run -d
-      --name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}"
+      --name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}"
       -e DATAFED_GLOBUS_APP_SECRET="GLOBUS_DATAFED_APP_SECRET"
       -e DATAFED_GLOBUS_APP_ID="$GLOBUS_DATAFED_APP_ID"
       -e DATAFED_ZEROMQ_SESSION_SECRET="$DATAFED_ZEROMQ_SESSION_SECRET"
@@ -310,13 +315,16 @@ end-to-end-core-setup:
     - docker
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
+    - mkdir -p "$HOST_LOG_FILE_PATH"
+    - chown gitlab-runner "$HOST_LOG_FILE_PATH"
     - echo "$BRANCH_LOWER"
     - ./scripts/generate_datafed.sh
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
     - ./scripts/container_stop.sh -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}"
+    - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
     - >
       docker run -d
-      --name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}"
+      --name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}"
       -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET"
       -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID"
       -e DATAFED_ZEROMQ_SESSION_SECRET="$CI_DATAFED_ZEROMQ_SESSION_SECRET"
@@ -337,26 +345,30 @@ end-to-end-repo-setup:
   variables:
     IMAGE_TAG: "dlsw/datafed/repo-"
     GIT_STRATEGY: clone
-    HOST_LOG_FILE_PATH: "/var/log/datafed/datafed-repo.log"
+    HOST_LOG_FILE_PATH: "$CI_PROJECT_DIR/logs"
+    CONTAINER_LOG_FILE_PATH: "/datafed/logs"
   stage: end-to-end-setup
   tags:
     - ci-datafed-globus
     - docker
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
+    - mkdir -p "$HOST_LOG_FILE_PATH"
+    - chown gitlab-runner "$HOST_LOG_FILE_PATH"
     - ./scripts/generate_datafed.sh
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
     - ./scripts/container_stop.sh -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}"
+    - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
     - >
       docker run -d
-      --name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}"
+      --name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}"
       -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET"
       -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID"
       -e DATAFED_ZEROMQ_SESSION_SECRET="$CI_DATAFED_ZEROMQ_SESSION_SECRET"
       -e DATAFED_ZEROMQ_SYSTEM_SECRET="$CI_DATAFED_ZEROMQ_SYSTEM_SECRET"
       -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN"
-      -v "$HOST_LOG_FILE_PATH:/var/log/datafed/datafed-ws.log"
+      -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}"
       -v "$CI_DATAFED_REPO_PUB_KEY":/source/install/keys/datafed-repo-key.pub
       -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv
       -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" 
@@ -365,6 +377,8 @@ end-to-end-repo-setup:
 end-to-end-gcs-authz-setup:
   variables:
     IMAGE_TAG: "dlsw/datafed/gcs-authz-"
+    HOST_LOG_FILE_PATH: "$CI_PROJECT_DIR/logs"
+    CONTAINER_LOG_FILE_PATH: "/datafed/logs"
     GIT_STRATEGY: clone
   stage: end-to-end-setup
   tags:
@@ -373,18 +387,21 @@ end-to-end-gcs-authz-setup:
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
+    - mkdir -p "$HOST_LOG_FILE_PATH"
+    - chown gitlab-runner "$HOST_LOG_FILE_PATH"
     - ./scripts/generate_datafed.sh
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567
     - ./scripts/container_stop.sh -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}"
+    - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
     - >
       docker run -d
-      --name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}"
+      --name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}"
       -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET"
       -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID"
       -e DATAFED_ZEROMQ_SESSION_SECRET="$CI_DATAFED_ZEROMQ_SESSION_SECRET"
       -e DATAFED_ZEROMQ_SYSTEM_SECRET="$CI_DATAFED_ZEROMQ_SYSTEM_SECRET"
       -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN"
-      -v "$HOST_LOG_FILE_PATH:/var/log/datafed/datafed-ws.log"
+      -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}"
       -v "$CI_DATAFED_REPO_PUB_KEY":/source/install/keys/datafed-repo-key.pub
       -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv
       -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" 
@@ -401,28 +418,27 @@ end-to-end-arango-setup:
     - ./scripts/run_arango_service.sh
 
 
-end-to-end-client-test:
-  variables:
-    GIT_STRATEGY: clone
-  stage: end-to-end-test
-  tags:
-    - ci-datafed-client
-  script:
-    - ./scripts/generate_datafed.sh
-    - env
-    - cat ./config/datafed.sh
-    - >
-      cmake -S. -B build
-      -DENABLE_FOXX_TESTS=OFF
-      -DBUILD_CORE_SERVER=OFF
-      -DBUILD_COMMON=OFF
-      -DBUILD_WEB_SERVER=OFF
-      -DBUILD_DOCS=OFF
-      -DBUILD_PYTHON_CLIENT=ON
-      -DBUILD_TESTS=ON
-      -DENABLE_END_TO_END_TESTS=ON
-      -DINSTALL_FOXX=OFF
-    - cmake --build build
-    - cmake --build build --target install
-
+#end-to-end-client-test:
+#  variables:
+#    GIT_STRATEGY: clone
+#  stage: end-to-end-test
+#  tags:
+#    - ci-datafed-client
+#  script:
+#    - ./scripts/generate_datafed.sh
+#    - env
+#    - cat ./config/datafed.sh
+#    - >
+#      cmake -S. -B build
+#      -DENABLE_FOXX_TESTS=OFF
+#      -DBUILD_CORE_SERVER=OFF
+#      -DBUILD_COMMON=OFF
+#      -DBUILD_WEB_SERVER=OFF
+#      -DBUILD_DOCS=OFF
+#      -DBUILD_PYTHON_CLIENT=ON
+#      -DBUILD_TESTS=ON
+#      -DENABLE_END_TO_END_TESTS=ON
+#      -DINSTALL_FOXX=OFF
+#    - cmake --build build
+#    - cmake --build build --target install
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,8 +1,21 @@
-## PATCH Bug Fixes/
-1. [915] - Refactor CI to use pipelines Gitlab feature as opposed to custom
-   solution
+# Pre-release
 
+## MAJOR Breaking changes
+
+## PATCH Bug fixes/Technical Debt/Documentation
+1. [914] - Improve GitHub template
+2. [910] - Static code analysis and status checks fixed, improvements to CI
+1. [915] - Refactor CI to use pipelines Gitlab feature along with pipelines 
+           script
+
+# v2023.10.23.15.50
+
+## MINOR Feature
+1. [906] - Added backup and cert refresh scripts. 
+
+## PATCH Bug Fixes/Technical Debt/Documentation
 1. [911] - Add GitHub template
+2. [913] - Fixed bug, when endpoint info returns an empty array check to seee if array is empty before accessing elements
 
 # v2023.8.21.10.40
 
@@ -19,7 +32,7 @@
 7. [879] - Added correlation ids to messages for tracking
 8. [879] - Split log output into server specific files.
 
-## PATCH Bug fixes/Technical Debt
+## PATCH Bug Fixes/Technical Debt/Documentation
 1. [879] - Fixed thread safety of repo list calls in core server which where causing
 seg faults
 2. [879] - Added better error reporting when attempting to delete repo with running tasks.