[Fixes #4521] Migrate Tests to CircleCI: smoke, apps, upload, analyti…

…cs and CSW (#6760) * - Test Fixes: CircleCI * - Test Fixes: CircleCI * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - Update .sh modes * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - TESTING: upload tests progress request errors * - TESTING: upload tests progress request errors * - TESTING: upload tests progress request errors * - Restore the whole suite: monitoring, integration and upload * - TESTING: upload tests progress request errors
GeoNode · Dec 30, 2020 · e357f55 · e357f55
1 parent 87b991d
commit e357f55
Show file tree

Hide file tree

Showing 16 changed files with 511 additions and 138 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -34,7 +34,7 @@ jobs:
 
       - run:
           name: Build the stack
-          command: docker-compose -f docker-compose.yml build
+          command: docker-compose -f docker-compose-test.yml build
           working_directory: ./
 
       - when:
@@ -54,7 +54,7 @@ jobs:
 
       - run:
           name: Start the stack
-          command: docker-compose -f docker-compose.yml up -d
+          command: docker-compose -f docker-compose-test.yml up -d
           working_directory: ./
 
       - run:
@@ -99,65 +99,81 @@ jobs:
             - run:
                 name: Run test suite
                 command: |
-                  docker-compose -f docker-compose.yml exec db psql -U postgres -c 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();'
-                  docker-compose -f docker-compose.yml exec db createdb -U postgres -T postgres test_postgres
-                  docker-compose -f docker-compose.yml exec db createdb -U postgres -T postgres test_geonode
-                  docker-compose -f docker-compose.yml exec db createdb -U postgres -T postgres test_geonode_data
-                  docker-compose -f docker-compose.yml exec db psql -U postgres -d test_geonode -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
-                  docker-compose -f docker-compose.yml exec db psql -U postgres -d test_geonode_data -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
-                  docker-compose -f docker-compose.yml exec django bash -c '<<parameters.test_suite>>'
+                  docker-compose -f docker-compose-test.yml exec db psql -U postgres -c 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();'
+                  docker-compose -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_postgres
+                  docker-compose -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode
+                  docker-compose -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode_data
+                  docker-compose -f docker-compose-test.yml exec db psql -U postgres -d test_geonode -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
+                  docker-compose -f docker-compose-test.yml exec db psql -U postgres -d test_geonode_data -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
+                  docker-compose -f docker-compose-test.yml exec django bash -c '<<parameters.test_suite>>'
                 working_directory: ./
             - run:
                 name: Run pep8 checks
                 command: |
-                  docker-compose -f docker-compose.yml exec django bash -c 'flake8 geonode'
-                  docker-compose -f docker-compose.yml exec django bash -c 'codecov; bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8'
+                  docker-compose -f docker-compose-test.yml exec django bash -c 'flake8 geonode'
+                  docker-compose -f docker-compose-test.yml exec django bash -c 'codecov; bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8'
                 working_directory: ./
 
 workflows:
 
   commit:
     jobs:
       - build:
-          name: geonode_test_suite
+          name: geonode_test_suite_smoke
           load_docker_cache: true
           save_docker_cache: true
-          test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb geonode.tests.smoke $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))")
+          test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb geonode.tests.smoke
+      - build:
+          name: geonode_test_suite
+          load_docker_cache: true
+          save_docker_cache: false
+          test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))")
+          requires:
+            - geonode_test_suite_smoke
       - build:
           name: geonode_test_integration_csw
           load_docker_cache: true
           save_docker_cache: false
           test_suite: ./test_csw.sh
           requires:
-            - geonode_test_suite
+            - geonode_test_suite_smoke
+      - build:
+          name: geonode_test_integration_upload
+          load_docker_cache: true
+          save_docker_cache: false
+          test_suite: ./test_upload.sh
+          requires:
+            - geonode_test_suite_smoke
       - build:
           name: geonode_test_integration_monitoring
           load_docker_cache: true
           save_docker_cache: false
           test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb geonode.tests.smoke geonode.monitoring.tests.integration
           requires:
-            - geonode_test_suite
+            - geonode_test_suite_smoke
+
+      # TODO
       # - build:
       #     name: tests_integration
       #     load_docker_cache: true
       #     save_docker_cache: false
       #     test_suite: 'geonode.tests.integration'
       #     requires:
-      #       - geonode_test_suite
+      #       - geonode_test_suite_smoke
       # - build:
       #     name: tests_geoserver_integration
       #     load_docker_cache: true
       #     save_docker_cache: false
       #     test_suite: 'geonode.geoserver.tests.integration'
       #     requires:
-      #       - geonode_test_suite
+      #       - geonode_test_suite_smoke
       # - build:
       #     name: tests_upload_integration
       #     load_docker_cache: true
       #     save_docker_cache: false
       #     test_suite: 'geonode.upload.tests.integration'
       #     requires:
-      #       - geonode_test_suite
+      #       - geonode_test_suite_smoke
 
   # nightly:
   #   triggers:

diff --git a/.env_test b/.env_test
@@ -0,0 +1,170 @@
+COMPOSE_PROJECT_NAME=geonode
+DOCKER_HOST_IP=
+DOCKER_ENV=production
+# See https://github.com/geosolutions-it/geonode-generic/issues/28
+# to see why we force API version to 1.24
+DOCKER_API_VERSION="1.24"
+BACKUPS_VOLUME_DRIVER=local
+
+C_FORCE_ROOT=1
+
+DEBUG=False
+
+DJANGO_SETTINGS_MODULE=geonode.settings
+GEONODE_INSTANCE_NAME=geonode
+GEONODE_LB_HOST_IP
+GEONODE_LB_PORT
+
+
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+GEONODE_DATABASE=geonode
+GEONODE_DATABASE_PASSWORD=geonode
+GEONODE_GEODATABASE=geonode_data
+GEONODE_GEODATABASE_PASSWORD=geonode_data
+GEONODE_DATABASE_SCHEMA=public
+GEONODE_GEODATABASE_SCHEMA=public
+DATABASE_URL=postgis://geonode:geonode@db:5432/geonode
+GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data
+GEONODE_DB_CONN_MAX_AGE=0
+GEONODE_DB_CONN_TOUT=5
+DEFAULT_BACKEND_DATASTORE=datastore
+BROKER_URL=amqp://guest:guest@rabbitmq:5672/
+ASYNC_SIGNALS=True
+
+
+SITEURL=http://localhost:8001/
+
+STATIC_ROOT=/mnt/volumes/statics/static/
+MEDIA_ROOT=/mnt/volumes/statics/uploaded/
+GEOIP_PATH=/mnt/volumes/statics/geoip.db
+
+ALLOWED_HOSTS=['django', '*']
+
+DEFAULT_BACKEND_UPLOADER=geonode.importer
+TIME_ENABLED=True
+MOSAIC_ENABLED=False
+HAYSTACK_SEARCH=False
+HAYSTACK_ENGINE_URL=http://elasticsearch:9200/
+HAYSTACK_ENGINE_INDEX_NAME=haystack
+HAYSTACK_SEARCH_RESULTS_PER_PAGE=200
+
+CACHE_BUSTING_STATIC_ENABLED=False
+CACHE_BUSTING_MEDIA_ENABLED=False
+
+MEMCACHED_ENABLED=False
+MEMCACHED_BACKEND=django.core.cache.backends.memcached.MemcachedCache
+MEMCACHED_LOCATION=127.0.0.1:11211
+MEMCACHED_LOCK_EXPIRE=3600
+MEMCACHED_LOCK_TIMEOUT=10
+
+MAX_DOCUMENT_SIZE=2
+CLIENT_RESULTS_LIMIT=5
+API_LIMIT_PER_PAGE=1000
+
+# HTTPD Server
+GEONODE_LB_HOST_IP=localhost
+GEONODE_LB_PORT=80
+
+# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only)
+# port where the server can be reached on HTTPS
+HTTP_HOST=localhost
+HTTPS_HOST=
+
+HTTP_PORT=8001
+HTTPS_PORT=443
+
+# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work
+# with an ip) and it must be reachable from the outside. This can be one of the following :
+# disabled : we do not get a certificate at all (a placeholder certificate will be used)
+# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates)
+# production : we get a normal certificate (default)
+LETSENCRYPT_MODE=disabled
+# LETSENCRYPT_MODE=staging
+# LETSENCRYPT_MODE=production
+
+RESOLVER=127.0.0.11
+
+# GIS Server
+GEOSERVER_WEB_UI_LOCATION=http://localhost:8001/geoserver/
+GEOSERVER_PUBLIC_LOCATION=http://localhost:8001/geoserver/
+GEOSERVER_LOCATION=http://geoserver:8080/geoserver/
+GEOSERVER_ADMIN_USER=admin
+GEOSERVER_ADMIN_PASSWORD=geoserver
+
+OGC_REQUEST_TIMEOUT=30
+OGC_REQUEST_MAX_RETRIES=1
+OGC_REQUEST_BACKOFF_FACTOR=0.3
+OGC_REQUEST_POOL_MAXSIZE=10
+OGC_REQUEST_POOL_CONNECTIONS=10
+
+# GIS Client
+GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore
+MAPBOX_ACCESS_TOKEN=
+BING_API_KEY=
+GOOGLE_API_KEY=
+
+# Monitoring
+MONITORING_ENABLED=True
+MONITORING_DATA_TTL=365
+USER_ANALYTICS_ENABLED=True
+USER_ANALYTICS_GZIP=True
+CENTRALIZED_DASHBOARD_ENABLED=False
+MONITORING_SERVICE_NAME=local-geonode
+MONITORING_HOST_NAME=geonode
+
+# Other Options/Contribs
+MODIFY_TOPICCATEGORY=True
+AVATAR_GRAVATAR_SSL=True
+AVATAR_DEFAULT_URL=/geonode/img/avatar.png
+
+EXIF_ENABLED=True
+CREATE_LAYER=True
+FAVORITE_ENABLED=True
+
+# #################
+# Security
+# #################
+# Admin Settings
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=admin
+ADMIN_EMAIL=admin@localhost
+
+# EMAIL Notifications
+EMAIL_ENABLE=False
+DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend
+DJANGO_EMAIL_HOST=localhost
+DJANGO_EMAIL_PORT=25
+DJANGO_EMAIL_HOST_USER=
+DJANGO_EMAIL_HOST_PASSWORD=
+DJANGO_EMAIL_USE_TLS=False
+DJANGO_EMAIL_USE_SSL=False
+DEFAULT_FROM_EMAIL='GeoNode <[email protected]>'
+
+# Session/Access Control
+LOCKDOWN_GEONODE=False
+CORS_ORIGIN_ALLOW_ALL=True
+X_FRAME_OPTIONS=ALLOW-FROM ALL
+SESSION_EXPIRED_CONTROL_ENABLED=True
+DEFAULT_ANONYMOUS_VIEW_PERMISSION=True
+DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True
+
+# Users Registration
+ACCOUNT_OPEN_SIGNUP=True
+ACCOUNT_EMAIL_REQUIRED=True
+ACCOUNT_APPROVAL_REQUIRED=False
+ACCOUNT_CONFIRM_EMAIL_ON_GET=False
+ACCOUNT_EMAIL_VERIFICATION=none
+ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False
+ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False
+ACCOUNT_AUTHENTICATION_METHOD=username_email
+AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True
+
+# OAuth2
+OAUTH2_API_KEY=
+OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV
+OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3
+
+# GeoNode APIs
+API_LOCKDOWN=False
+TASTYPIE_APIKEY=
diff --git a/.travis.yml b/.travis.yml
@@ -105,49 +105,6 @@ jobs:
           - SESSION_EXPIRED_CONTROL_ENABLED: 'True'
           - CELERY_ALWAYS_EAGER: 'True'
 
-    - stage: upload-tests
-      name: "GeoServer-backend Upload Tests"
-      virtualenv:
-          system_site_packages: false
-      env:
-          - BACKEND: 'geonode.geoserver'
-          - DOCKER_COMPOSE_VERSION: 1.19.0
-          - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
-          - GEOSERVER_SERVER_PORT: 8080
-          - ON_TRAVIS: 'True'
-          - TEST_RUNNER_KEEPDB: 'True'
-          - TEST_RUN_INTEGRATION: 'True'
-          - TEST_RUN_INTEGRATION_SERVER: 'False'
-          - TEST_RUN_INTEGRATION_UPLOAD: 'True'
-          - TEST_RUN_INTEGRATION_MONITORING: 'False'
-          - TEST_RUN_INTEGRATION_CSW: 'False'
-          - TEST_RUN_INTEGRATION_BDD: 'False'
-          - MONITORING_ENABLED: 'False'
-          - SESSION_EXPIRED_CONTROL_ENABLED: 'True'
-          - CELERY_ALWAYS_EAGER: 'True'
-
-    - stage: anlytics-tests
-      name: "GeoServer-backend Monitoring Tests"
-      virtualenv:
-          system_site_packages: false
-      env:
-          - BACKEND: 'geonode.geoserver'
-          - DOCKER_COMPOSE_VERSION: 1.19.0
-          - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
-          - GEOSERVER_SERVER_PORT: 8080
-          - ON_TRAVIS: 'True'
-          - TEST_RUNNER_KEEPDB: 'True'
-          - TEST_RUN_INTEGRATION: 'True'
-          - TEST_RUN_INTEGRATION_SERVER: 'False'
-          - TEST_RUN_INTEGRATION_UPLOAD: 'False'
-          - TEST_RUN_INTEGRATION_MONITORING: 'True'
-          - TEST_RUN_INTEGRATION_CSW: 'False'
-          - TEST_RUN_INTEGRATION_BDD: 'False'
-          - MONITORING_ENABLED: 'True'
-          - USER_ANALYTICS_ENABLED: 'True'
-          - SESSION_EXPIRED_CONTROL_ENABLED: 'True'
-          - CELERY_ALWAYS_EAGER: 'True'
-
     - stage: integration-tests
       name: "Backend Integration Tests"
       virtualenv:

diff --git a/celery.sh b/celery.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 nohup celery -A geonode.celery_app:app beat -l DEBUG -f /var/log/celery.log &>/dev/null &
-nohup celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log &>/dev/null &
+nohup celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=2 -n worker1@%h -f /var/log/celery.log &>/dev/null &
 nohup celery -A geonode.celery_app:app flower --auto_refresh=True --debug=False --broker=${BROKER_URL} --basic_auth=${ADMIN_USERNAME}:${ADMIN_PASSWORD} --address=0.0.0.0 --port=5555 &>/dev/null &
diff --git a/celery_dev.sh b/celery_dev.sh
@@ -2,4 +2,4 @@ set -a
 . ./.env_dev
 set +a
 
-celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=10 -n worker1@%h
+celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=2 -n worker1@%h