.gitlab-ci.yml

# default installed image for docker executor is: python:3.6
# using an image that can do git, docker, docker-compose
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner:latest

# build directory inside
# https://gitlab.mpcdf.mpg.de/help/ci/runners/configure_runners.md#custom-build-directories

# https://docs.gitlab.com/ee/ci/yaml/workflow.html
# https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
# if: CI_COMMIT_BRANCH && CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000"
# A branch pipeline, but it is the first commit for that branch
# if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
# For an existing workflow section to switch from branch pipelines to merge request pipelines when a merge request is created.
# if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
# A branch pipeline, but a merge request is open for that branch, do not run the branch pipeline.
# if: $CI_PIPELINE_SOURCE == "merge_request_event"
# A merge request pipeline, start the pipeline.
# if: $CI_COMMIT_BRANCH
# A branch pipeline, but there is no merge request open for the branch, run the branch pipeline.

variables:
  DOCKER_TAG: ${CI_COMMIT_REF_SLUG}

workflow:
  rules:
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000"
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_BRANCH
    - if: $CI_COMMIT_TAG
      variables:
        DOCKER_TAG: ${CI_COMMIT_REF_NAME}
    - when: never

stages:
  - build
  - test
  - deploy
  - release

update changelog:
  stage: build
  script:
    - curl -X POST "https://gitlab.mpcdf.mpg.de/api/v4/projects/$CI_PROJECT_ID/repository/changelog?version=${CI_COMMIT_TAG:1}&access_token=$CI_ACCESS_TOKEN&message=Add%20changelog%20for%20version%20$CI_COMMIT_TAG%20%5Bskip-ci%5D"
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/

build:
  stage: build
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
    GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4
    DOCKER_BUILDKIT: 1
  before_script:
    - docker --version
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
  script:
    - docker build -t ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} .
    - docker build --target dev_node -t ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}  .
    - docker build --target dev_python -t ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}  .
    - docker push ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker push ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}
    - docker push ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  rules:
    - when: on_success

python linting:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app
  script:
    - pycodestyle --config=pycodestyle.ini nomad tests
    - pylint --rcfile=.pylintrc nomad tests
    - mypy nomad tests
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

gui linting:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app/gui
  script:
    - yarn run lint
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success


check python dependencies:
  stage: test
  image: python:3.9
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
    GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4
  before_script:
    - pip install --upgrade pip-tools
  script:
    - scripts/check_python_dependencies.sh
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: manual
      allow_failure: true


check gui artifacts:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app
  script:
    - scripts/check_gui_artifacts.sh
  after_script:
    - >
      if [ $CI_JOB_STATUS == 'failed' ]; then
        echo 'Make sure that the right GUI artifacts have been regenerated and committed by running'
        echo './scripts/generate_gui_artifacts.sh'
      fi
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success


python tests:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  services:
    - name: rabbitmq:3.11.5
      alias: rabbitmq
    - name: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
      alias: elastic
      command:
        - bash
        - "-c"
        - ES_JAVA_OPTS="-Xms512m -Xmx512m" docker-entrypoint.sh elasticsearch -Ediscovery.type=single-node -Expack.security.enabled=false
    - name: mongo:5.0.6
      alias: mongo
  variables:
    GIT_STRATEGY: none
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
    RABBITMQ_DEFAULT_PASS: rabbitmq
    RABBITMQ_DEFAULT_VHOST: /
    NOMAD_RABBITMQ_HOST: rabbitmq
    NOMAD_ELASTIC_HOST: elastic
    NOMAD_MONGO_HOST: mongo
    NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
    NOMAD_NORMALIZE_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.msg
  before_script:
    - sleep 5
    - curl http://elastic:9200/_cat/health
    - cd /app
  script:
    - python -m pytest --cov=nomad --cov-report term --cov-report xml:coverage.xml --cov-config=.coveragerc -sv tests
  after_script:
    - cd /app
    - cp coverage.xml $CI_PROJECT_DIR
  coverage: /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
  artifacts:
    when: always
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

gui tests:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app/gui
  script:
    - yarn test --ci --collectCoverage --reporters=default --reporters=jest-junit --silent
      --testPathIgnorePatterns=src/components/entry/ArchiveEntryView.spec.js
      --testPathIgnorePatterns=src/components/archive/FileBrowser.spec.js
      --testPathIgnorePatterns=src/components/archive/MetainfoBrowser.spec.js

  after_script:
    - cd /app/gui
    - cp junit.xml $CI_PROJECT_DIR
    - cp coverage/cobertura-coverage.xml $CI_PROJECT_DIR
  timeout: 1h 30m
  coverage: /Lines\s*:\s*(\d+.?\d*)%/
  artifacts:
    when: always
    reports:
      junit:
        - junit.xml
      coverage_report:
        coverage_format: cobertura
        path: cobertura-coverage.xml
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success


deploy dev:
  stage: deploy
  environment:
    name: dev/$CI_COMMIT_REF_NAME
    deployment_tier: development
    url: https://nomad-lab.eu/dev/rae/${CI_ENVIRONMENT_SLUG}
    auto_stop_in: 7 days
    on_stop: stop deploy dev
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_CONFIG} | base64 -d > ~/.kube/config
    - echo ${CI_ENVIRONMENT_SLUG}
    - echo "https://nomad-lab.eu/dev/rae/${CI_ENVIRONMENT_SLUG}"
  script:
    - helm dependency update ops/kubernetes/nomad
    - helm upgrade ${CI_ENVIRONMENT_SLUG} ops/kubernetes/nomad
      --install
      --namespace nomad
      --values ops/kubernetes/deployments/dev-values.yaml
      --set proxy.external.path=/dev/rae/${CI_ENVIRONMENT_SLUG}
      --set jupyterhub.hub.baseUrl=/dev/rae/${CI_ENVIRONMENT_SLUG}/north
      --set jupyterhub.fullnameOverride=${CI_ENVIRONMENT_SLUG}-north
      --set jupyterhub.singleuser.podNameTemplate="${CI_ENVIRONMENT_SLUG}-north-{username}--{servername}"
      --set jupyterhub.hub.config.GenericOAuthenticator.oauth_callback_url=https://nomad-lab.eu/dev/rae/${CI_ENVIRONMENT_SLUG}/north/hub/oauth_callback
      --set image.tag=${DOCKER_TAG}
      --set roll=true
      --timeout=15m
      --wait
      --cleanup-on-fail
  needs:
    - job: build
  rules:
    - when: manual
      allow_failure: true

stop deploy dev:
  stage: deploy
  variables:
    GIT_STRATEGY: none
  environment:
    name: dev/$CI_COMMIT_REF_NAME
    action: stop
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_CONFIG} | base64 -d > ~/.kube/config
  script:
    - helm uninstall ${CI_ENVIRONMENT_SLUG} --namespace nomad
  needs:
    - job: build
  rules:
    - when: manual
      allow_failure: true

deploy prod staging:
  stage: deploy
  environment:
    name: staging
    deployment_tier: production
    url: https://nomad-lab.eu/prod/v1/staging
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ~/.kube/config
  script:
    - helm dependency update ops/kubernetes/nomad
    - helm upgrade nomad-staging-v1 ops/kubernetes/nomad
      --install
      --values ops/kubernetes/deployments/prod-staging-values.yaml
      --set image.tag=${DOCKER_TAG}
      --set roll=true
      --timeout=15m
      --wait
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      nomad client -n https://nomad-lab.eu/prod/v1/staging/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
  rules:
    - when: manual
      allow_failure: true

deploy prod test:
  stage: deploy
  environment:
    name: test
    deployment_tier: production
    url: https://nomad-lab.eu/prod/v1/test
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ~/.kube/config
  script:
    - helm dependency update ops/kubernetes/nomad
    - helm upgrade nomad-test-v1 ops/kubernetes/nomad
      --install
      --values ops/kubernetes/deployments/prod-test-values.yaml
      --set image.tag=${DOCKER_TAG}
      --set roll=true
      --timeout=15m
      --wait
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      nomad client -n https://nomad-lab.eu/prod/v1/test/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
  rules:
    - when: manual
      allow_failure: true

deploy prod util:
  stage: deploy
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ~/.kube/config
  script:
    - helm dependency update ops/kubernetes/nomad
    - helm upgrade nomad-util-v1 ops/kubernetes/nomad
      --install
      --values ops/kubernetes/deployments/prod-util-values.yaml
      --set image.tag=${DOCKER_TAG}
      --set roll=true
      --timeout=15m
      --wait
  rules:
    - when: manual
      allow_failure: true

deploy prod develop:
  stage: deploy
  environment:
    name: develop
    deployment_tier: production
    url: https://nomad-lab.eu/prod/v1/develop
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ~/.kube/config
  script:
    - helm dependency update ops/kubernetes/nomad
    - helm upgrade nomad-develop-v1 ops/kubernetes/nomad
      --install
      --namespace default
      --values ops/kubernetes/deployments/prod-develop-values.yaml
      --set image.tag=${DOCKER_TAG}
      --set roll=true
      --timeout=15m
      --wait
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      nomad client -n https://nomad-lab.eu/prod/v1/develop/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
  rules:
    - if: $CI_COMMIT_BRANCH == "develop" && $NIGHTLY
    - when: manual
      allow_failure: true

release latest image:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
  script:
    - docker tag ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} ${CI_REGISTRY_IMAGE}:latest
    - docker push ${CI_REGISTRY_IMAGE}:latest
  rules:
    - when: manual
      allow_failure: true

release stable image:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
  script:
    - docker tag ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} ${CI_REGISTRY_IMAGE}:stable
    - docker push ${CI_REGISTRY_IMAGE}:stable
  rules:
    - if: $CI_COMMIT_TAG
      when: manual
      allow_failure: true
    - when: never

python package:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
    - docker pull ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  script:
    - docker run --rm ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab-*.tar.gz
  rules:
    - when: manual
      allow_failure: true


pypi package:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
    - docker pull ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  script:
    - docker run --rm ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/nomad-lab-*.tar.gz
  rules:
    - if: $CI_COMMIT_TAG
      when: manual
      allow_failure: true
    - when: never

push to github:
  stage: release
  script:
    - git checkout ${CI_COMMIT_REF_NAME}
    - git push "https://${CI_GITHUB_ACCESS_TOKEN}@github.com/nomad-coe/nomad.git" ${CI_COMMIT_REF_NAME}
  rules:
    - if: $CI_COMMIT_BRANCH == "develop" || $CI_COMMIT_TAG