Merge pull request #915 from ORNL/JoshuaSBrown-switch-to-pipelines

Attempt to fix bottleneck and resource contention

.gitlab-ci.yml

@@ -1,35 +1,68 @@
 
 stages:
-  - setup
+  - ci-infrastructure-check
+  - build-infrastructure
+  - signal
   - clear-docker-cache
   - build-deploy-base
+  - provision-client
   - build-unit-test-deploy
   - end-to-end-setup
   - end-to-end-test
   - end-to-end-teardown
 
 ################################################################################
-# STAGE: setup
+# STAGE: ci-infrastructure-check
 ################################################################################
-# Setup stage is designed to check that the infrastructure is up and running
-# before attempting to launch the CI pipelines
-setup-vms:
-  stage: setup
+# ci-infrastructure-check stage is designed to check that the infrastructure is
+# up and running before attempting to launch the CI pipelines
+
+check-ci-infrastructure:
+  stage: ci-infrastructure-check
   tags:
     - datafed-infrastructure
   script:
     - ./scripts/ci_pipeline_setup.sh --compute-instance-name "ci-datafed-arangodb"
     - ./scripts/ci_pipeline_setup.sh --compute-instance-name "ci-datafed-core"
     - ./scripts/ci_pipeline_setup.sh --compute-instance-name "ci-datafed-globus2"
     - ./scripts/ci_pipeline_setup.sh --compute-instance-name "ci-datafed-client"
+  resource_group: infrastructure_build
+
+run-trigger-job:
+  stage: build-infrastructure
+  rules:
+    - when: on_failure
+  trigger:
+    project: 8f4/datafedci
+    branch: main
+    strategy: depend
+  resource_group: infrastructure_build
 
+################################################################################
+# STAGE: signal
+################################################################################
+# Stage is used to separte the trigger job from the remaining jobs and to act
+# as an anchor for setting up dependencies
+signal:
+  stage: signal
+  tags:
+    - runner
+  script:
+    - echo "Starting Build"
+  rules:
+    - exists:
+      - check-ci-infrastrucure
+    - exists:
+      - run-trigger-job
+    - when: on_success
 
 ################################################################################
 # STAGE: clear-docker-cache
 ################################################################################
 # Used to clear out the cache on VMs where the images are being built
 clear-core-cache:
   stage: clear-docker-cache
+  needs: ["signal"]
   tags:
     - ci_1
   script:
@@ -39,6 +72,7 @@ clear-core-cache:
 
 clear-repo-cache:
   stage: clear-docker-cache
+  needs: ["signal"]
   tags:
     - ci_3
   script:
@@ -128,6 +162,20 @@ build-gcs-base:
     - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG3} .
     - docker push code.ornl.gov:4567/${IMAGE_TAG3}
 
+################################################################################
+# STAGE: provision client
+################################################################################
+provision-client:
+  needs: ["signal"]
+  variables:
+    IMAGE_TAG: "dlsw/datafed/repo-base"
+    GIT_STRATEGY: clone
+  stage: provision-client
+  tags:
+    - ci-datafed-client
+  script:
+    - ./scripts/install_client_dependencies.sh
+
 ################################################################################
 # STAGE: build
 ################################################################################
@@ -191,7 +239,6 @@ build-foxx:
     DATAFED_DATABASE_PASSWORD: "${CI_DATAFED_DATABASE_PASSWORD}"
     DATAFED_ZEROMQ_SYSTEM_SECRET: "${CI_DATAFED_ZEROMQ_SYSTEM_SECRET}"
   stage: build-unit-test-deploy
-  needs: ["setup-vms"]
   tags:
     - ci-datafed-arango
   rules:
@@ -303,6 +350,7 @@ end-to-end-ws-setup:
       -v "${CI_DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${CERT_FILE_NAME}"
       -v "${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}"
       -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}"
+       </dev/null &>/dev/null &
     - ./scripts/container_run_test.sh -e -c "1" -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" 
 
 # Note here we use network=host to run the docker container this is
@@ -342,6 +390,7 @@ end-to-end-core-setup:
       -v "$CI_DATAFED_CORE_PUB_KEY":/source/install/keys/datafed-core-key.pub
       -v "$CI_DATAFED_CORE_PRIV_KEY":/source/install/keys/datafed-core-key.priv
       -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" 
+       </dev/null &>/dev/null &
     - ./scripts/container_run_test.sh -e -c "1" -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" 
 
 
@@ -376,6 +425,7 @@ end-to-end-repo-setup:
       -v "$CI_DATAFED_REPO_PUB_KEY":/source/install/keys/datafed-repo-key.pub
       -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv
       -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" 
+       </dev/null &>/dev/null &
     - ./scripts/container_run_test.sh -e -c "1" -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" 
 
 end-to-end-gcs-authz-setup:
@@ -409,6 +459,7 @@ end-to-end-gcs-authz-setup:
       -v "$CI_DATAFED_REPO_PUB_KEY":/source/install/keys/datafed-repo-key.pub
       -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv
       -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" 
+       </dev/null &>/dev/null &
     - ./scripts/container_run_test.sh -e -c "1" -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" 
 
 end-to-end-arango-setup:
@@ -421,10 +472,24 @@ end-to-end-arango-setup:
     - arangod --version
     - ./scripts/run_arango_service.sh
 
-
+################################################################################
+# Actual Testing
+################################################################################
 #end-to-end-client-test:
 #  variables:
 #    GIT_STRATEGY: clone
+#    DATAFED_DATABASE_HOST: "${CI_DATAFED_DATABASE_HOST}"
+#    DATAFED_DATABASE_ZEROMQ_SYSTEM_SECRET: "${CI_DATAFED_DATABASE_ZEROMQ_SYSTEM_SECRET}"
+#    DATAFED_DATABASE_PASSWORD: "${CI_DATAFED_DATABASE_PASSWORD}"
+#    DATAFED_USER89_PASSWORD: "${CI_DATAFED_USER89_PASSWORD}"
+#    DATAFED_USER89_GLOBUS_REFRESH_TOKEN: "${CI_DATAFED_USER89_GLOBUS_REFRESH_TOKEN}"
+#    DATAFED_USER89_GLOBUS_ACCESS_TOKEN: "${CI_DATAFED_USER89_GLOBUS_ACCESS_TOKEN}"
+#    DATAFED_USER89_GLOBUS_UUID: "${CI_DATAFED_USER89_GLOBUS_UUID}"
+#    DATAFED_USER99_PASSWORD: "${CI_DATAFED_USER99_PASSWORD}"
+#    DATAFED_USER99_GLOBUS_REFRESH_TOKEN: "${CI_DATAFED_USER99_GLOBUS_REFRESH_TOKEN}"
+#    DATAFED_USER99_GLOBUS_ACCESS_TOKEN: "${CI_DATAFED_USER99_GLOBUS_ACCESS_TOKEN}"
+#    DATAFED_USER99_GLOBUS_UUID: "${CI_DATAFED_USER99_GLOBUS_UUID}"
+#    DATAFED_ZEROMQ_SYSTEM_SECRET: "${CI_DATAFED_ZEROMQ_SYSTEM_SECRET}"
 #  stage: end-to-end-test
 #  tags:
 #    - ci-datafed-client
@@ -444,5 +509,6 @@ end-to-end-arango-setup:
 #      -DENABLE_END_TO_END_TESTS=ON
 #      -DINSTALL_FOXX=OFF
 #    - cmake --build build
-#    - cmake --build build --target install
-
+#    - cmake --build build --target pydatafed
+#    - cmake --build build --target test
+#

CHANGELOG.md

@@ -10,6 +10,7 @@
 2. [910] - Static code analysis and status checks fixed, improvements to CI
 3. [923] - Fixed missing flag in certificate refresh script
 4. [917] - Add additional files to .gitignore
+5. [915] - Refactor CI to use pipelines Gitlab feature along with pipelines 
 
 # v2023.10.23.15.50
 

core/database/tests/test_foxx.sh

@@ -125,10 +125,24 @@ export NVM_DIR="$HOME/.nvm"
 
 nvm use $NODE_VERSION
 
+FOXX_PREFIX=""
+{
+	# Determine if exists globally first
+	which foxx
+} || {
+	FOXX_PREFIX="~/bin/"
+}
+
 PATH_TO_PASSWD_FILE=${SOURCE}/database_temp.password
 if [ "$TEST_TO_RUN" == "all" ]
 then
-  foxx test -u ${local_DATABASE_USER} -p ${PATH_TO_PASSWD_FILE} --database ${local_DATABASE_NAME} /api/${local_FOXX_MAJOR_API_VERSION} --reporter spec
+  "${FOXX_PREFIX}foxx" test -u "${local_DATABASE_USER}" \
+    -p "${PATH_TO_PASSWD_FILE}" \
+    --database "${local_DATABASE_NAME}" \
+    "/api/${local_FOXX_MAJOR_API_VERSION}" --reporter spec
 else
-  foxx test -u ${local_DATABASE_USER} -p ${PATH_TO_PASSWD_FILE} --database ${local_DATABASE_NAME} /api/${local_FOXX_MAJOR_API_VERSION} "$TEST_TO_RUN" --reporter spec
+  "${FOXX_PREFIX}foxx" test -u "${local_DATABASE_USER}" \
+    -p "${PATH_TO_PASSWD_FILE}" \
+    --database "${local_DATABASE_NAME}" \
+    "/api/${local_FOXX_MAJOR_API_VERSION}" "$TEST_TO_RUN" --reporter spec
 fi

python/datafed_pkg/requirements.txt

@@ -0,0 +1,5 @@
+protobuf>=3,<=3.20
+pyzmq>=16
+wget>=3
+click>=7
+prompt_toolkit>=2

python/datafed_pkg/setup.py

@@ -9,6 +9,9 @@
     long_description = f.read()
 
 
+with open("requirements.txt", "r") as f:
+    install_requires = [line.strip() for line in f]
+
 setuptools.setup(
     name="datafed",
     version=__version__,
@@ -20,13 +23,7 @@
     url="https://github.com/ORNL/DataFed",
     packages=setuptools.find_packages(),
     setup_requires=["setuptools"],
-    install_requires=[
-        "protobuf>=3, <=3.20",
-        "pyzmq>=16",
-        "wget>=3",
-        "click>=7",
-        "prompt_toolkit>=2",
-    ],
+    install_requires=install_requires,
     entry_points={"console_scripts": ["datafed = datafed.CLI:run"]},
     classifiers=[
         "Programming Language :: Python :: 3",